Merge branch 'master' into api/op_benchmark_base

Author: Xreki

PaddleNLP

@@ -57,7 +57,7 @@ function prepare(){
     apt-get install libmysqlclient20=5.7.33-0ubuntu0.16.04.1 --allow-downgrades
     apt-get install libmysqlclient-dev git curl psmisc -y
     pip install MySQL-python
-
+    pip install shyaml
 
     save_log_dir=${all_path}/logs/${paddle_version}/${implement_type}
 
@@ -86,12 +86,21 @@ function prepare(){
     cd ${ROOT_PATH}
     rm -rf *
 
-    git clone https://github.com/PaddlePaddle/benchmark.git --recursive
-    echo "****************${implement_type} prepare had done*****************"
-
+    git clone https://github.com/PaddlePaddle/benchmark.git
     cd ${BENCHMARK_ROOT}
     benchmark_commit_id=$(git log|head -n1|awk '{print $2}')
     echo "benchmark_commit_id is: "${benchmark_commit_id}
+    
+    init_group="paddle_group"     # 可配到任务参数里
+    repo_list=`cat submodule.yaml | shyaml get-value ${init_group}`
+    echo $repo_list
+    for i in ${repo_list[@]}
+    do
+        git submodule init $i
+        git submodule update $i
+    done
+    echo "*******************init submodule done******************************" 
+    echo "****************${implement_type} prepare had done*****************"
 
     # 动态图升级到cuda10.1 python3.7，静态图切cuda10.1 python3.7
     if [[ 'dynamic_graph' == ${implement_type} ]] || [[ 'static_graph' == ${implement_type} ]] || [[ 'dynamic_to_static' == ${implement_type} ]]; then

scripts/auto_run_paddle.sh

@@ -72,9 +72,10 @@ dy_video_TimeSformer(){
     echo "dy_video_TimeSformer"
     cur_model_path=${BENCHMARK_ROOT}/PaddleVideo/
     cd ${cur_model_path}/benchmark/TimeSformer/
+    pip install -r requirements.txt
     pip install scikit-image==0.18.2
     pip install pooch==1.5.2
-	bash run_all.sh local
+    bash run_all.sh local
     rm -rf ${BENCHMARK_ROOT}/PaddleVideo/    # 避免数据集占用docker内过多空间,在执行最后一个模型后删掉
 }
 
@@ -256,6 +257,7 @@ dy_tsn(){
     cur_model_path=${BENCHMARK_ROOT}/PaddleVideo
     cd ${cur_model_path}
 
+    pip install -r requirements.txt
     pip install wget av
     pip install scikit-image==0.18.2
     pip install pooch==1.5.2
@@ -345,6 +347,7 @@ dy_seg(){
 dy_slowfast(){
     cur_model_path=${BENCHMARK_ROOT}/PaddleVideo
     cd ${cur_model_path}
+    pip install -r requirements.txt
     pip install tqdm
     pip install decord
     pip install pandas av
@@ -461,6 +464,7 @@ dy_tsm(){
     cur_model_path=${BENCHMARK_ROOT}/PaddleVideo
     cd ${cur_model_path}
 
+    pip install -r requirements.txt
     pip install wget av decord
     pip install scikit-image==0.18.2
     pip install pooch==1.5.2

scripts/dynamic_graph_models.sh

@@ -34,6 +34,7 @@ function prepare(){
     cp ${all_path}/sources.list /etc/apt
     apt-get update
     apt-get install libmysqlclient-dev git curl psmisc -y
+    pip install shyaml
 
     save_log_dir=${log_dir}
 
@@ -59,12 +60,21 @@ function prepare(){
     mkdir -p ${ROOT_PATH}
     cd ${ROOT_PATH}
     rm -rf benchmark
-    git clone https://github.com/PaddlePaddle/benchmark.git --recursive
-
+    git clone https://github.com/PaddlePaddle/benchmark.git
     cd ${BENCHMARK_ROOT}
     benchmark_commit_id=$(git log|head -n1|awk '{print $2}')
     echo "benchmark_commit_id is: "${benchmark_commit_id}
 
+    init_group="paddle_group"     # 可配到任务参数里
+    repo_list=`cat submodule.yaml | shyaml get-value ${init_group}`
+    echo $repo_list
+    for i in ${repo_list[@]}
+    do
+        git submodule init $i
+        git submodule update $i
+    done
+    echo "*******************init submodule done******************************"
+
     rm -rf run_env
     mkdir run_env
     ln -s $(which python3.7) run_env/python

scripts/pr_run_paddle.sh

@@ -0,0 +1,34 @@
+paddle_group:
+    models
+    PaddleClas
+    PaddleDetection
+    PaddleSeg
+    PaddleSpeech
+    PaddleOCR
+    PaddleNLP
+    PaddleVideo
+    PaddleGAN
+other_frame_group:    
+    OtherFrame/video/PyTorch/models/TimeSformer
+    OtherFrame/nlp/PyTorch/scripts/NLP/gpt/Megatron-LM
+    OtherFrame/nlp/PyTorch/apex
+    OtherFrame/Speech/PyTorch/wenet/models/wenet
+    OtherFrame/seg/PyTorch/models/mmseg
+    OtherFrame/clas/mxnet/models/gluon-cv
+    OtherFrame/clas/PyTorch/models/mmclassification
+    OtherFrame/clas/PyTorch/models/HRNet-Image-Classification
+    OtherFrame/clas/PyTorch/models/Twins
+    OtherFrame/gan/PyTorch/mmedting/models/mmedi
+    OtherFrame/ocr/PyTorch/models/PSENet
+    OtherFrame/nlp/PyTorch/models/xlnet/transformers
+    OtherFrame/detection/PyTorch/models/jde
+    OtherFrame/detection/PyTorch/models/fairmot
+    OtherFrame/Speech/PyTorch/PWGAN/models/Parakeet/PWGAN/ParallelWaveGAN
+    OtherFrame/detection/PyTorch/models/mmdetection
+    OtherFrame/detection/PyTorch/models/SOLO
+    OtherFrame/detection/PyTorch/models/mmpose
+    OtherFrame/nlp/PyTorch/models/xlnet
+    OtherFrame/gan/PyTorch/fomm/models/fomm
+    OtherFrame/gan/PyTorch/styleganv2/models/styleganv2
+    OtherFrame/ocr/PyTorch/models/DB
+    OtherFrame/ocr/TensorFlow/models/EAST