Skip to content

Commit caf6732

Browse files
committed
Revert "Add more timing metrics to SpmdTrainer"
GitOrigin-RevId: bb74e60
1 parent 8c1516c commit caf6732

1 file changed

Lines changed: 1 addition & 22 deletions

File tree

axlearn/common/trainer.py

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -601,12 +601,7 @@ def run(
601601
while True:
602602
self._maybe_record_event(measurement.Event.START_DATA_LOADING)
603603
try:
604-
input_batch_start_time = time.perf_counter()
605604
input_batch = next(input_iterator)
606-
self.summary_writer(
607-
self.step + 1, # log this at the same step as other metrics below
608-
{"step_time/input_batch": time.perf_counter() - input_batch_start_time},
609-
)
610605
self._maybe_record_event(measurement.Event.END_DATA_LOADING)
611606
logging.log_first_n(
612607
logging.INFO, "host_input_batch=%s", 3, utils.shapes(input_batch)
@@ -631,13 +626,8 @@ def run(
631626
)
632627
self.vlog(3, "Done step %s", self.step)
633628
num_steps += 1
634-
635-
# Record timing metrics
636-
now = time.perf_counter()
637-
self.summary_writer(
638-
self.step, {"step_time/total": now - input_batch_start_time}
639-
)
640629
if num_steps % 100 == 0:
630+
now = time.perf_counter()
641631
average_step_time = (now - start_time) / num_steps
642632
self._step_log("Average step time: %s seconds", average_step_time)
643633
self.summary_writer(self.step, {"average_step_time": average_step_time})
@@ -1119,13 +1109,7 @@ def _run_step(
11191109
trainer_state=self.trainer_state, input_batch=input_batch, with_xsc=run_with_xsc
11201110
)
11211111
# Run the compiled function.
1122-
train_step_start_time = time.perf_counter()
11231112
self._trainer_state, outputs = compiled_train_step_fn(self.trainer_state, input_batch)
1124-
# Wait for step to finish
1125-
outputs["loss"].block_until_ready()
1126-
self.summary_writer(
1127-
self.step, {"step_time/train_step": time.perf_counter() - train_step_start_time}
1128-
)
11291113

11301114
if self.step % 100 == 0 or 0 <= self.step <= 5:
11311115
self._step_log(
@@ -1141,12 +1125,7 @@ def _run_step(
11411125
)
11421126

11431127
# Checkpointer policy will decide if we should save.
1144-
save_checkpoint_start_time = time.perf_counter()
11451128
self.save_checkpoint(evaler_summaries=evaler_summaries)
1146-
self.summary_writer(
1147-
self.step,
1148-
{"step_time/save_checkpoint": time.perf_counter() - save_checkpoint_start_time},
1149-
)
11501129

11511130
return_dict = {"loss": outputs["loss"], "aux": outputs["aux"]}
11521131
# Returns evaler_summaries if force_run_evals is not None or empty set.

0 commit comments

Comments
 (0)