Skip to content

Commit b019008

Browse files
hysunflowerhysunflower
authored
add pse scripts, test=document_fix (#1253)
* add pse scripts, test=document_fix * add pse scripts, test=document_fix * add ocr db models, test=document_fix * add ocr db models, test=document_fix * add ocr db models, test=document_fix
1 parent 03d6d6b commit b019008

8 files changed

Lines changed: 100 additions & 37 deletions

File tree

‎OtherFrame/ocr/PyTorch/DB_scripts/pre_ENV.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
## 安装依赖
4+
5+
git clone https://github.com/LDOUBLEV/DBNet.pytorch.git
6+
cd DBNet.pytorch
7+
pip3.7 install -r requirement.txt
8+
9+
# 准备数据
10+
bash benchmark/prepare_data.sh
11+

‎OtherFrame/ocr/PyTorch/DB_scripts/readme.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ benchmark目录下的文件用于获取并分析DB.pytorch的训练日志，运
99
安装相关依赖和下载数据
1010

1111
```
12-
git clone https://github.com/LDOUBLEV/DBNet.pytorch
12+
git clone https://github.com/LDOUBLEV/DBNet.pytorch.git
1313
cd DBNet.pytorch
1414
pip3.7 install -r requirement.txt
1515
bash benchmark/prepare_data.sh

‎OtherFrame/ocr/PyTorch/DB_scripts/run_benchmark.sh‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33
# bash benchmark/run_benchmark.sh sp
44
# bash benchmark/run_benchmark.sh mp
55

6-
model_name="pytorch_db_res18"
7-
86
mode=$1
97

108
batch_list=(8 16)
@@ -13,18 +11,19 @@ for batch in ${batch_list[@]}; do
1311

1412
if [ ${mode} = "sp" ]; then
1513
#export CUDA_VISIBLE_DEVICES=0
16-
train_cmd="python tools/train.py --config_file config/icdar2015_resnet18_FPN_DBhead_polyLR_bs${batch}.yaml"
14+
train_cmd="python3.7 tools/train.py --config_file config/icdar2015_resnet18_FPN_DBhead_polyLR_bs${batch}.yaml"
1715
num_gpu_devices=1
1816
else
1917
#export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
20-
train_cmd="python -m torch.distributed.launch --nproc_per_node=8 tools/train.py --config_file config/icdar2015_resnet18_FPN_DBhead_polyLR_bs${batch}.yaml"
18+
train_cmd="python3.7 -m torch.distributed.launch --nproc_per_node=8 tools/train.py --config_file config/icdar2015_resnet18_FPN_DBhead_polyLR_bs${batch}.yaml"
2119
num_gpu_devices=8
2220
fi
2321

2422
echo $train_cmd
2523
run_log_path=${TRAIN_LOG_DIR:-$(pwd)}
2624
fp_item="fp32"
27-
log_file="${run_log_path}/${model_name}_${mode}_bs${batch}_${fp_item}_${num_gpu_devices}"
25+
model_name=det_res18_db_v2.0_bs${batch}_${fp_item}
26+
log_file="${run_log_path}/${model_name}_${mode}_${num_gpu_devices}"
2827
echo $log_file
2928

3029
timeout 15m ${train_cmd} > ${log_file} 2>&1
@@ -36,15 +35,15 @@ for batch in ${batch_list[@]}; do
3635
echo -e "${model_name}, SUCCESS"
3736
export job_fail_flag=0
3837
fi
39-
kill -9 `ps -ef|grep 'python'|awk '{print $2}'`
38+
kill -9 `ps -ef|grep 'python3.7'|awk '{print $2}'`
4039

4140
if [ $mode = "mp" -a -d mylog ]; then
4241
rm ${log_file}
4342
cp mylog/workerlog.0 ${log_file}
4443
fi
4544

46-
analysis_cmd="python benchmark/analysis.py --filename ${log_file} --mission_name ${model_name} --run_mode ${mode} --direction_id 0 --keyword 'speed:' --base_batch_size ${batch} --skip_steps 1 --gpu_num ${num_gpu_devices} --index 1 --model_mode=-1 --ips_unit=samples/sec"
47-
eval $analysis_cmd
45+
analysis_cmd="python3.7 benchmark/analysis.py --filename ${log_file} --mission_name OCR --model_name ${model_name} --run_mode ${mode} --direction_id 0 --keyword 'speed:' --base_batch_size ${batch} --skip_steps 1 --gpu_num ${num_gpu_devices} --index 1 --model_mode=-1 --ips_unit=samples/sec"
46+
eval $analysis_cmd > ${model_name}_${num_gpu_devices}.speed 2>&1
4847
done
4948

5049

‎OtherFrame/ocr/PyTorch/PrepareEnv.sh‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,9 @@ echo "*******prepare benchmark***********"
55

66
################################# 创建一些log目录,如:
77
export BENCHMARK_ROOT=/workspace
8-
log_date=`date "+%Y.%m%d.%H%M%S"`
9-
frame=pytorch1.7.1
10-
cuda_version=10.2
11-
save_log_dir=${BENCHMARK_ROOT}/logs/${frame}_${log_date}_${cuda_version}/
12-
13-
if [[ -d ${save_log_dir} ]]; then
14-
rm -rf ${save_log_dir}
15-
fi
168
# this for update the log_path coding mat
17-
export TRAIN_LOG_DIR=${save_log_dir}/train_log
18-
mkdir -p ${TRAIN_LOG_DIR}
199

20-
log_path=${TRAIN_LOG_DIR}
10+
log_path=${TRAIN_LOG_DIR:-"pwd"}
2111

2212
################################# 安装框架 如:
2313
pip3.7 install -U pip -i https://mirror.baidu.com/pypi/simple
@@ -26,7 +16,8 @@ pip3.7 install pqi -i https://mirror.baidu.com/pypi/simple
2616
pqi add baidu https://mirror.baidu.com/pypi/simple
2717
pqi use baidu
2818

19+
#pip3.7 install torch==1.10.0+cu111 torchvision==0.11.2+cu111 torchaudio==0.10.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html
2920
pip3.7 install torch==1.7.1+cu101 torchvision==0.8.2+cu101 torchaudio==0.7.2 -f https://download.pytorch.org/whl/torch_stable.html
30-
21+
pip3.7 install anyconfig
3122
echo "*******prepare benchmark end***********"
3223

‎OtherFrame/ocr/PyTorch/run_PyTorch.sh‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,41 @@
11
#!/usr/bin/env bash
22

3-
ImageName="registry.baidubce.com/paddlepaddle/paddle:2.0.2-gpu-cuda10.1-cudnn7";
3+
ImageName="paddlepaddle/paddle:latest-dev-cuda11.2-cudnn8-gcc82"
44
docker pull ${ImageName}
55

66
# 启动镜像后测试PSE模型
77
run_cmd="cd /workspace;
88
\cp -f /workspace/PrepareEnv.sh ./;
99
bash PrepareEnv.sh;
10+
1011
cd /workspace/models/PSENet/;
1112
pip3.7 install -r requirement.txt;
1213
\cp -f /workspace/scripts/PSENet/preData.sh ./;
13-
bash preData.sh;
1414
\cp -f /workspace/scripts/PSENet/run_benchmark.sh ./;
1515
\cp -f /workspace/scripts/PSENet/analysis_log.py ./;
16-
CUDA_VISIBLE_DEVICES=0 bash run_benchmark.sh sp 8 fp32 2;
17-
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh mp 64 fp32 2;
16+
bash preData.sh;
17+
sed -i '/set\ -xe/d' run_benchmark.sh
18+
CUDA_VISIBLE_DEVICES=0 bash run_benchmark.sh sp 8 fp32 2 > ocr_pse_bs8_fp32_sp.speed 2>&1;
19+
sleep 60
20+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh mp 64 fp32 2 > ocr_pse_bs8_fp32_mp.speed 2>&1;
21+
sleep 60
22+
CUDA_VISIBLE_DEVICES=0 bash run_benchmark.sh sp 16 fp32 2 > ocr_pse_bs16_fp32_sp.speed 2>&1;
23+
sleep 60
24+
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 bash run_benchmark.sh mp 128 fp32 2 > ocr_pse_bs16_fp32_mp.speed 2>&1;
1825
"
1926

20-
nvidia-docker run --name test_torch_ocr -i --net=host --shm-size=128g -v $PWD:/workspace ${ImageName} /bin/bash -c "${run_cmd}"
27+
nvidia-docker run --name test_torch_ocr -i \
28+
--net=host \
29+
--shm-size=128g \
30+
-v $PWD:/workspace \
31+
-v /ssd3:/ssd3 \
32+
-v /ssd2:/ssd2 \
33+
-e "ALL_PATH=${all_path}" \
34+
-v "BENCHMARK_ROOT=/workspace" \
35+
-e "http_proxy=${http_proxy}" \
36+
-e "https_proxy=${http_proxy}" \
37+
-e "no_proxy=bcebos.com" \
38+
${ImageName} /bin/bash -c "${run_cmd}"
2139

2240
nvidia-docker stop test_torch_ocr
2341
nvidia-docker rm test_torch_ocr

‎OtherFrame/ocr/PyTorch/run_PyTorch_DB.sh‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
#!/usr/bin/env bash
2+
3+
ImageName="registry.baidubce.com/paddlepaddle/paddle:2.0.2-gpu-cuda10.1-cudnn7"
4+
#ImageName="paddlepaddle/paddle:latest-dev-cuda11.2-cudnn8-gcc82"
5+
docker pull ${ImageName}
6+
7+
# 启动镜像后测试DB模型
8+
run_cmd="cd /workspace;
9+
bash PrepareEnv.sh;
10+
cp /workspace/DB_scripts/* /workspace/models/DB/
11+
cd /workspace/models/DB/
12+
pip3.7 install -r requirement.txt
13+
bash prepare_data.sh
14+
bash run_benchmark.sh sp
15+
sleep 60
16+
bash run_benchmark.sh mp
17+
"
18+
19+
nvidia-docker run --name test_torch_ocr -i \
20+
--net=host \
21+
--shm-size=128g \
22+
-v $PWD:/workspace \
23+
-v /ssd3:/ssd3 \
24+
-v /ssd2:/ssd2 \
25+
-e "ALL_PATH=${all_path}" \
26+
-v "BENCHMARK_ROOT=/workspace" \
27+
-e "http_proxy=${http_proxy}" \
28+
-e "https_proxy=${http_proxy}" \
29+
-e "no_proxy=bcebos.com" \
30+
${ImageName} /bin/bash -c "${run_cmd}"
31+
32+
nvidia-docker stop test_torch_ocr
33+
nvidia-docker rm test_torch_ocr

‎OtherFrame/ocr/PyTorch/scripts/PSENet/run_benchmark.sh‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ function _set_params(){
55
run_mode=${1:-"sp"}
66
batch_size=${2:-"64"}
77
fp_item=${3:-"fp32"} # fp32|fp16
8-
max_iter=${4:-"10"} # 如果需要修改代码提前中断
9-
run_log_path=${TRAIN_LOG_DIR:-$(pwd)}
8+
max_epoch=${4:-"10"} # 如果需要修改代码提前中断
9+
run_log_path=$(pwd)
1010

11-
model_name="ocr_PSENet"
11+
model_name="det_r50_vd_pse"_bs${batch_size}_${fp_item}
1212
mission_name="OCR" # 模型所属任务名称，具体可参考scripts/config.ini 必填）
1313
direction_id=0 # 任务所属方向，0：CV，1：NLP，2：Rec。 (必填)
1414
ips_unit="samples/sec"
@@ -26,7 +26,7 @@ function _train(){
2626
rm -rf checkpoints/
2727
# 如需开启特殊优化flag、参数请注明
2828

29-
train_cmd="config/psenet/psenet_r50_ic15_736.py --cfg-options data.batch_size=${batch_size} train_cfg.epoch=${max_iter} model.backbone.pretrained=False"
29+
train_cmd="config/psenet/psenet_r50_ic15_736.py --cfg-options data.batch_size=${batch_size} train_cfg.epoch=${max_epoch} model.backbone.pretrained=False"
3030
case ${run_mode} in
3131
sp) train_cmd="python3.7 train.py "${train_cmd} ;;
3232
mp)
@@ -42,20 +42,15 @@ function _train(){
4242
echo -e "${model_name}, SUCCESS"
4343
export job_fail_flag=0
4444
fi
45-
46-
if [ $run_mode = "mp" -a -d mylog ]; then
47-
rm ${log_file}
48-
cp mylog/workerlog.0 ${log_file}
49-
fi
5045
}
5146

5247
function _analysis_log(){
53-
analysis_cmd="python3.7 analysis_log.py --filename ${log_file} --mission_name ${model_name} --run_mode ${run_mode} --direction_id 0 --keyword 'ips:' --base_batch_size ${batch_size} --skip_steps 1 --gpu_num ${num_gpu_devices} --index 1 --model_mode=-1 --ips_unit=samples/sec"
48+
analysis_cmd="python3.7 analysis_log.py --filename ${log_file} --model_name ${model_name} --mission_name ${mission_name} --run_mode ${run_mode} --direction_id 0 --keyword 'ips:' --base_batch_size ${batch_size} --skip_steps 1 --gpu_num ${num_gpu_devices} --index 1 --model_mode=-1 --ips_unit=images/sec"
5449
eval $analysis_cmd
5550
}
5651

5752
function _kill_process(){
58-
kill -9 `ps -ef|grep 'python3.7'|awk '{print $2}'`
53+
kill -9 `ps -ef|grep 'python3.7'|awk '{print $2}'` >/dev/null 2>&1
5954
}
6055
_set_params $@
6156
_train

‎OtherFrame/scripts/auto_run.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,22 @@ detec_torch_fast(){
113113
cp models/SOLO/*fp32_8 ${TRAIN_LOG_DIR}
114114
}
115115

116+
ocr_torch_ocr(){
117+
# pse
118+
cur_model_path=${ROOT_DIR}/ocr/PyTorch
119+
cd ${cur_model_path}
120+
bash run_PyTorch.sh
121+
mv ${cur_model_path}/models/PSENet/*speed ${LOG_PATH_INDEX_DIR}
122+
mv ${cur_model_path}/models/PSENet/det* ${TRAIN_LOG_DIR}
123+
124+
# DB
125+
cur_model_path=${ROOT_DIR}/ocr/PyTorch
126+
cd ${cur_model_path}
127+
bash run_PyTorch_DB.sh
128+
mv ${cur_model_path}/models/DB/*speed ${LOG_PATH_INDEX_DIR}
129+
mv ${cur_model_path}/models/DB/*sp ${TRAIN_LOG_DIR}
130+
mv ${cur_model_path}/models/DB/*mp ${TRAIN_LOG_DIR}
131+
}
116132
set_env
117133
for model_name in ${cur_torch_list[@]}
118134
do

0 commit comments

Comments
 (0)