Merge task config template, and distributed MxNet training changes to Amol's fork

benchmark_runner.py

@@ -31,13 +31,14 @@
     parser.add_argument('--num-gpus', type=int, help='Numbers of gpus. e.g. --num-gpus 8')
     parser.add_argument('--command-to-execute', type=str, help='The script command that performs benchmarking')
     parser.add_argument('--data-set', type=str, help='The data set to use for benchmarking, eg. imagenet')
+    parser.add_argument('--metrics-template', type=str, help='The template file to use for metrics pattern', default=CONFIG_TEMPLATE)
 
     args = parser.parse_args()    
 
     if(args.data_set == 'imagenet'): 
         data_manager.getImagenetData()
 
-    config.read(CONFIG_TEMPLATE)
+    config.read(args.metrics_template)
 
     for name, value in config.items(args.metrics_policy):
         if(name == 'patterns'):

image_classification/common/fit.py

@@ -23,13 +23,19 @@
 import math
 import mxnet as mx
 
+def get_epoch_size(args, kv):
+    if 'dist' in args.kv_store:
+        nworkers = kv.num_workers
+    else:
+        nworkers = 1
+    return math.ceil(int(args.num_examples / nworkers) / args.batch_size)
 
 def _get_lr_scheduler(args, kv):
     if 'lr_factor' not in args or args.lr_factor >= 1:
         return (args.lr, None)
-    epoch_size = args.num_examples / args.batch_size
-    if 'dist' in args.kv_store:
-        epoch_size /= kv.num_workers
+
+    epoch_size = get_epoch_size(args, kv)
+
     begin_epoch = args.load_epoch if args.load_epoch else 0
     if 'pow' in args.lr_step_epochs:
         lr = args.lr
@@ -156,8 +162,17 @@ def fit(args, network, data_loader, **kwargs):
     logging.basicConfig(level=logging.DEBUG, format=head)
     logging.info('start with arguments %s', args)
 
+    epoch_size = get_epoch_size(args, kv)
+
     # data iterators
     (train, val) = data_loader(args, kv)
+
+    if 'dist' in args.kv_store and not 'async' in args.kv_store:
+        logging.info('Resizing training data to %d batches per machine', epoch_size)
+        # resize train iter to ensure each machine has same number of batches per epoch
+        # if not, dist_sync can hang at the end with one machine waiting for other machines
+        train = mx.io.ResizeIter(train, epoch_size)
+
     if args.test_io:
         tic = time.time()
         for i, batch in enumerate(train):
@@ -213,13 +228,11 @@ def fit(args, network, data_loader, **kwargs):
     # A limited number of optimizers have a warmup period
     has_warmup = {'lbsgd', 'lbnag'}
     if args.optimizer in has_warmup:
+
         if 'dist' in args.kv_store:
             nworkers = kv.num_workers
         else:
             nworkers = 1
-        epoch_size = args.num_examples / args.batch_size / nworkers
-        if epoch_size < 1:
-            epoch_size = 1
         macrobatch_size = args.macrobatch_size
         if macrobatch_size < args.batch_size * nworkers:
             macrobatch_size = args.batch_size * nworkers

task_config_template.cfg

@@ -1,3 +1,9 @@
+
+[metrics_parameters_inference]
+patterns = ['Prediction-Time: (\d+\.\d+|\d+) milliseconds']
+metrics = ['prediction_time']
+compute_method = ['average']
+
 [metrics_parameters_imperative_hybrid_top_1]
 patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
 metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
@@ -69,6 +75,13 @@ compute_method = ['average', 'last', 'last', 'total']
 command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --gpus 8 --epochs 20 --log-interval 50 --kvstore device
 num_gpus = 8
 
+[dawnbench_cifar10_symbolic]
+patterns = ['Epoch \d+, Batch \d+, Speed=(\d+\.\d+|\d+)', 'Epoch \d+, Training accuracy=(\d+\.\d+|\d+)', 'Epoch \d+, Validation accuracy=(\d+\.\d+|\d+)', 'Epoch \d+, Duration=(\d+\.\d+|\d+)']
+metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
+compute_method = ['average', 'last', 'last', 'total']
+command_to_execute = python dawnbench/cifar10.py --gpus 4 --early-stopping-acc 0.86 --epochs 400 --lr 0.05 --total-batch-size 256
+num_gpus = 4
+
 [lstm_ptb_imperative]
 patterns = ['time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'test loss (\d+\.\d+|\d+)', 'test ppl (\d+\.\d+|\d+)']
 metrics = ['total_training_time', 'validation_loss', 'validation_perplexity', 'test_loss', 'test_perplexity']
@@ -87,7 +100,7 @@ num_gpus = 8
 patterns = ['Time cost=(\d+\.\d+|\d+)', 'Train-perplexity=(\d+\.\d+|\d+)', 'Validation-perplexity=(\d+\.\d+|\d+)', 'Speed: (\d+\.\d+|\d+) samples/sec']
 metrics = ['total_training_time', 'train_perplexity', 'validation_perplexity', 'speed']
 compute_method = ['total', 'last', 'last', 'average']
-command_to_execute = python word_language_model/lstm_bucketing.py --num-hidden 650 --num-embed 650 --gpus 8 --epochs 25 --kvstore device
+command_to_execute = python word_language_model/lstm_bucketing.py --num-hidden 650 --num-embed 650 --gpus 8 --epochs 25 --kv-store device
 num_gpus = 8
 
 [resnet50_imagenet_symbolic_fp16_batch_size32_p3_16]
@@ -153,6 +166,16 @@ compute_method = ['average', 'last', 'total','last']
 command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 512 --data-nthreads 15 --num-epochs 80
 num_gpus = 1
 
+[metrics_parameters_distributed_top_k]
+patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Train-top_k_accuracy_\d=(\d+\.\d+|\d+)','Validation-top_k_accuracy_\d=(\d+\.\d+|\d+)']
+metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time','training_acc_top5','validation_acc_top5']
+compute_method = ['average_aggregate', 'last', 'last', 'total','last','last']
+
+[metrics_parameters_distributed]
+patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)']
+metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
+compute_method = ['average_aggregate', 'last', 'last', 'total']
+
 [tensorflow_resnet50_p3_2xlg]
 patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
 metrics = ['Images per sec', 'Time']

utils/data_manager.py

@@ -5,8 +5,10 @@
 def getImagenetData():
 
 
+    os.system('mkdir -p data')
     if not os.path.exists('./data/imagenet1k-train.rec'):
-        os.system("wget -q https://s3.amazonaws.com/ragab-datasets/imagenet2012/imagenet1k-train.rec -P data/")
-
+        os.system("aws s3 cp  s3://imagenet-rec/imagenet1k-train.rec data/")
+    
     if not os.path.exists('./data/imagenet1k-val.rec'):
-        os.system("wget -q https://s3.amazonaws.com/ragab-datasets/imagenet2012/imagenet1k-val.rec -P data/")
+        os.system("aws s3 cp  s3://imagenet-rec/imagenet1k-val.rec data/")
+

utils/metrics_manager.py

@@ -17,12 +17,20 @@
 class BenchmarkMetricComputeMethod:
     @staticmethod
     def compute(metric_compute_method, metric):
+        numWorkers = int(os.getenv('DEEPLEARNING_WORKERS_COUNT','0'))
+
         if metric_compute_method == 'average':
             return 1.0 * sum(metric) / len(metric)
         elif metric_compute_method == 'last':
             return metric[-1]
         elif metric_compute_method == 'total':
-            return sum(metric)
+            if numWorkers == 0:
+                return sum(metric)
+            else:
+                return 1.0 * sum(metric) / numWorkers
+        elif metric_compute_method == 'average_aggregate':
+            assert numWorkers != 0
+            return (1.0 * sum(metric) / len(metric)) * numWorkers
         else:
             raise utils.errors.MetricComputeMethodError("This metric compute method is not supported!")