apply review suggestions

Signed-off-by: Hongwei Chen <hongweichen@microsoft.com>

Author: hwchen2017

training/DeepSpeed-Domino/domino/training.py

@@ -460,7 +460,6 @@ def train(forward_step_func, model, optimizer, opt_param_scheduler,
                 time_list.append(ite_time)
 
 
-            # ite_time = timers('ite-time').elapsed(barrier=True)
             if iteration % args.log_interval == 0 and is_rank_0():
                 loss = loss_dict['lm loss'].item()
                 print( 'iteration: {} | loss: {:.3f} | iteration time (ms): {:.2f} '.format(iteration, loss, ite_time))

training/DeepSpeed-Domino/pretrain_gpt.py

@@ -64,16 +64,6 @@ def forward_step(data_iterator, model):
         data_iterator)
     timers('batch-generator').stop()
     output_tensor = model(tokens, position_ids, attention_mask, labels=labels)
-    # start = torch.cuda.Event(enable_timing=True)
-    # end = torch.cuda.Event(enable_timing=True)
-    # start.record()
-    # output_tensor = model(tokens, position_ids, attention_mask, labels=labels)
-    # end.record()
-    # torch.cuda.synchronize()
-
-    # if is_rank_0():
-    #     print('forward time: ')
-    #     print(start.elapsed_time(end))
 
     return output_tensor, partial(loss_func, loss_mask)
 

training/DeepSpeed-Domino/pretrain_gpt3_13b.sh

@@ -13,9 +13,9 @@ NODE_RANK=0
 WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
 
 
-VOCAB_FILE="/home/code/dataset/gpt2-vocab.json"
-MERGE_FILE="/home/code/dataset/gpt2-merges.txt"
-DATA_PATH="/home/code/dataset/BookCorpusDataset_text_document"
+VOCAB_FILE="dataset/gpt2-vocab.json"
+MERGE_FILE="dataset/gpt2-merges.txt"
+DATA_PATH="dataset/BookCorpusDataset_text_document"
 
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"

training/DeepSpeed-Domino/pretrain_gpt3_6.7b.sh

@@ -13,9 +13,9 @@ NODE_RANK=0
 WORLD_SIZE=$(($GPUS_PER_NODE*$NNODES))
 
 
-VOCAB_FILE="/home/code/dataset/gpt2-vocab.json"
-MERGE_FILE="/home/code/dataset/gpt2-merges.txt"
-DATA_PATH="/home/code/dataset/BookCorpusDataset_text_document"
+VOCAB_FILE="dataset/gpt2-vocab.json"
+MERGE_FILE="dataset/gpt2-merges.txt"
+DATA_PATH="dataset/BookCorpusDataset_text_document"
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 SCRIPT_DIR="$SCRIPT_DIR/Megatron-LM"