File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -374,8 +374,9 @@ def tee_print(*a, **kw):
374374 model .train ()
375375 ema_model .train ()
376376 data_iter = iter (dataloader )
377- t_start = time .time ()
378377 global_step = start_step
378+ session_start_step = start_step
379+ t_start = time .time ()
379380
380381 while global_step < stop_step :
381382 try :
@@ -417,7 +418,8 @@ def tee_print(*a, **kw):
417418 # --- Logging ---
418419 if global_step % 100 == 0 :
419420 elapsed = time .time () - t_start
420- steps_per_sec = global_step / elapsed if elapsed > 0 else 0
421+ session_steps = global_step - session_start_step
422+ steps_per_sec = session_steps / elapsed if elapsed > 0 else 0
421423 eta_sec = (stop_step - global_step ) / steps_per_sec if steps_per_sec > 0 else 0
422424 writer .add_scalar ("train/loss" , loss .item (), global_step )
423425 writer .add_scalar ("train/grad_norm" , grad_norm , global_step )
You can’t perform that action at this time.
0 commit comments