End on correct epoch

Author: SuperLinguini

task_config_template.cfg

@@ -121,7 +121,7 @@ num_gpus = 4
 patterns = ['time = (\d+\.\d+|\d+)', 'wps = (\d+\.\d+|\d+)', 'train loss = (\d+\.\d+|\d+)', 'ppl = (\d+\.\d+|\d+)', 'log_perplexity = (\d+\.\d+|\d+)', 'perplexity = (\d+\.\d+|\d+)']
 metrics = ['total_training_time', 'words_per_second', 'train_loss', 'train_perplexity', 'test_loss', 'test_perplexity']
 compute_method = ['total', 'average', 'last', 'last', 'last', 'last']
-command_to_execute = python tensorflow_benchmark/tf_word_language_model/single_lm_train.py --gpus=4 --epochs=5 --datadir=tensorflow_benchmark/tf_word_language_model/1-billion-word-language-modeling-benchmark-r13output/ --hpconfig run_profiler=False,float16_rnn=False,num_steps=20,num_shards=8,num_layers=1,learning_rate=0.2,max_grad_norm=1,keep_prob=0.9,emb_size=1024,projected_size=512,state_size=2048,num_sampled=8192,batch_size=256
+command_to_execute = for i in {1..5}; do python tensorflow_benchmark/tf_word_language_model/single_lm_train.py --gpus=4 --epochs=1 --datadir=tensorflow_benchmark/tf_word_language_model/1-billion-word-language-modeling-benchmark-r13output/ --hpconfig run_profiler=False,float16_rnn=False,num_steps=20,num_shards=8,num_layers=1,learning_rate=0.2,max_grad_norm=1,keep_prob=0.9,emb_size=1024,projected_size=512,state_size=2048,num_sampled=8192,batch_size=256; python tensorflow_benchmark/tf_word_language_model/single_lm_train.py --logdir=/home/ubuntu/deep-learning-benchmark-mirror/lm1b --num_gpus=1 --mode=eval_full --datadir=tensorflow_benchmark/tf_word_language_model/1-billion-word-language-modeling-benchmark-r13output/ --hpconfig run_profiler=False,float16_rnn=False,num_steps=20,num_shards=8,num_layers=1,learning_rate=0.2,max_grad_norm=1,keep_prob=0.9,emb_size=1024,projected_size=512,state_size=2048,num_sampled=8192,batch_size=1; done
 num_gpus = 4
 
 [resnet50_imagenet_symbolic_fp16_batch_size32_p3_16]

tensorflow_benchmark/tf_word_language_model/run_utils.py

@@ -68,19 +68,19 @@ def run_train(dataset, hps, logdir, ps_device, eval_dataset, task=0, master=""):
 
             if epoch_done:
                 cur_epoch = int(x)
-                eval_data_iterator = eval_dataset.iterate_once(hps.batch_size * hps.num_gpus, hps.num_steps)
-                loss_nom = 0.0
-                loss_den = 0.0
-
-                for i, (x, y) in enumerate(eval_data_iterator):
-                    loss = sess.run(model.loss, {model.x: x, model.y: y})
-                    loss_nom += loss
-                    loss_den += 1
-                    loss = loss_nom / loss_den
-
-                log_perplexity = loss_nom / loss_den
-                print("Results after epoch %d: log_perplexity = %.3f perplexity = %.3f" % (
-                    cur_epoch, log_perplexity, np.exp(log_perplexity)))
+                # eval_data_iterator = eval_dataset.iterate_once(hps.batch_size * hps.num_gpus, hps.num_steps)
+                # loss_nom = 0.0
+                # loss_den = 0.0
+                #
+                # for i, (x, y) in enumerate(eval_data_iterator):
+                #     loss = sess.run(model.loss, {model.x: x, model.y: y})
+                #     loss_nom += loss
+                #     loss_den += 1
+                #     loss = loss_nom / loss_den
+                #
+                # log_perplexity = loss_nom / loss_den
+                # print("Results after epoch %d: log_perplexity = %.3f perplexity = %.3f" % (
+                #     cur_epoch, log_perplexity, np.exp(log_perplexity)))
 
                 x, y = next(data_iterator)
 

utils/cfg_process.py

@@ -51,7 +51,7 @@ def generate_cfg(cfg_template, cfg_path, **infra_spec):
                 elif "num_gpus" in infra_spec:
                     cmd = re.sub("--gpus=\d+", "", cmd)
                 if "epochs" in infra_spec and infra_spec["epochs"] is not None and infra_spec["epochs"] > 0:
-                    cmd = re.sub("--epochs=\d+", "--epochs=%d" % infra_spec["epochs"], cmd)
+                    cmd = re.sub("{1..\d+}", "{1..%d}" % infra_spec["epochs"], cmd)
             else:
                 if "num_gpus" in infra_spec and infra_spec["num_gpus"] is not None and infra_spec["num_gpus"] > 0:
                     cmd = re.sub("--gpus \d+", "--gpus %d" % infra_spec["num_gpus"], cmd)