hwmodule-tf2oda-server/tf2oda_train_eval_export_TEMPLATE.sh at main · embedded-machine-learning/hwmodule-tf2oda-server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
#!/bin/bash

echo "#==============================================#"
echo "# CDLEML Tool TF2 Object Detection API Training"
echo "#==============================================#"

# Constant Definition
USERNAME=wendt
USEREMAIL=alexander.wendt@tuwien.ac.at
#MODELNAME=tf2oda_efficientdetd0_320_240_coco17_pedestrian_all_LR002
#PYTHONENV=tf24
BASEPATH=`pwd`
SCRIPTPREFIX=../../eml-tools


#Extract model name from this filename
MYFILENAME=`basename "$0"`
MODELNAME=`echo $MYFILENAME | sed 's/tf2oda_train_eval_export_//' | sed 's/.sh//'`
echo Selected model: $MODELNAME


# Environment preparation
echo Activate environment $PYTHONENV
#call conda activate %PYTHONENV%
. ./init_env.sh

echo Activate task spooler
. ./init_ts.sh


echo "Start training of $MODELNAME on EDA02 $(date +"%Y%m%d %T")" | mail -s "Start Train $MODELNAME EDA02 $(date +"%Y%m%d %T")" $USEREMAIL


echo "#====================================#"
echo "#Train model"
echo "#====================================#"
echo model used $MODELNAME

python $SCRIPTPREFIX/training/tf2oda_model_main_training.py \
--pipeline_config_path=$BASEPATH/jobs/$MODELNAME.config \
--model_dir=$BASEPATH/models/$MODELNAME

echo "#====================================#"
echo "#Evaluate trained model"
echo "#====================================#"
echo Evaluate checkpoint performance in tensorboard

python $SCRIPTPREFIX/training/tf2oda_evaluate_ckpt_performance.py \
--pipeline_config_path=$BASEPATH/jobs/$MODELNAME.config \
--model_dir=$BASEPATH/models/$MODELNAME \
--checkpoint_dir=$BASEPATH/models/$MODELNAME


echo Read TF Summary from Tensorboard file
python $SCRIPTPREFIX/training/tf2oda_read_tf_summary.py \
--checkpoint_dir=$BASEPATH/models/$MODELNAME \
--out_dir=results/$MODELNAME/metrics

echo "#====================================#"
echo "#Export inference graph"
echo "#====================================#"
echo Export model $MODELNAME
python $SCRIPTPREFIX/training/tf2oda_export_savedmodel.py \
--input_type="image_tensor" \
--pipeline_config_path=$BASEPATH/jobs/$MODELNAME.config \
--trained_checkpoint_dir=$BASEPATH/models/$MODELNAME \
--output_directory=exported-models/$MODELNAME

echo "#====================================#"
echo "#Copy Exported Graph to Pickup folder"
echo "#====================================#"
mkdir tmp
cp -ar exported-models/$MODELNAME tmp


echo "Stop training of $MODELNAME on EDA02 $(date +"%Y%m%d %T")" | mail -s "Stop Train $MODELNAME EDA02 $(date +"%Y%m%d %T")" $USEREMAIL

echo "#======================================================#"
echo "# Training, evaluation and export of the model completed"
echo "#======================================================#"