Skip to content

Commit 8b58738

Browse files
fix: correct resumed training eta
1 parent c00c35f commit 8b58738

1 file changed

Lines changed: 4 additions & 2 deletions

File tree

training/ddpm-cifar10/train_ddpm_cifar10.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,9 @@ def tee_print(*a, **kw):
374374
model.train()
375375
ema_model.train()
376376
data_iter = iter(dataloader)
377-
t_start = time.time()
378377
global_step = start_step
378+
session_start_step = start_step
379+
t_start = time.time()
379380

380381
while global_step < stop_step:
381382
try:
@@ -417,7 +418,8 @@ def tee_print(*a, **kw):
417418
# --- Logging ---
418419
if global_step % 100 == 0:
419420
elapsed = time.time() - t_start
420-
steps_per_sec = global_step / elapsed if elapsed > 0 else 0
421+
session_steps = global_step - session_start_step
422+
steps_per_sec = session_steps / elapsed if elapsed > 0 else 0
421423
eta_sec = (stop_step - global_step) / steps_per_sec if steps_per_sec > 0 else 0
422424
writer.add_scalar("train/loss", loss.item(), global_step)
423425
writer.add_scalar("train/grad_norm", grad_norm, global_step)

0 commit comments

Comments
 (0)