Skip to content

Commit 5178d95

Browse files
committed
Fix stateful dataloader state not restored on resume after estimated_stepping_batches
Accessing trainer.estimated_stepping_batches in configure_optimizers (e.g. for OneCycleLR) runs FitLoop.setup_data() during strategy setup, before the checkpoint's loop state is restored. The second setup_data() call in fit_loop.run() then early-returns because the combined loader already exists, so the stateful dataloader states from the checkpoint are never loaded and resumed training replays already-seen batches. Load the pending dataloader states in the early-return path and recreate the fetcher iterator so the restored state takes effect. Fixes #20550
1 parent fe6b1cc commit 5178d95

3 files changed

Lines changed: 20 additions & 3 deletions

File tree

src/lightning/pytorch/CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
3232

3333
- Fixed `LightningModule.toggle_optimizer` / `untoggle_optimizer` breaking under `torch.compile` by disabling Dynamo tracing on these bookkeeping helpers ([#21513](https://github.com/Lightning-AI/pytorch-lightning/issues/21513))
3434

35+
- Fixed stateful dataloaders not restoring their state from a checkpoint when `trainer.estimated_stepping_batches` is accessed in `configure_optimizers` ([#20550](https://github.com/Lightning-AI/pytorch-lightning/issues/20550))
36+
3537
---
3638

3739
## [2.6.4] - 2026-05-20

src/lightning/pytorch/loops/fit_loop.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,13 @@ def run(self) -> None:
224224

225225
def setup_data(self) -> None:
226226
if self._combined_loader is not None and not self._should_reload_train_dl:
227+
# the dataloaders may have been set up before the checkpoint was loaded, e.g., if the user accessed
228+
# `trainer.estimated_stepping_batches` in `configure_optimizers`. in that case, restore the dataloader
229+
# states now and recreate the iterator so the restored states take effect
230+
if self.restarting and self._combined_loader_states_to_load:
231+
self._load_combined_loader_states()
232+
assert self._data_fetcher is not None
233+
iter(self._data_fetcher) # creates the iterator inside the fetcher
227234
return
228235

229236
trainer = self.trainer

tests/tests_pytorch/loops/test_loops.py

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1248,8 +1248,9 @@ def load_state_dict(self, state_dict):
12481248
),
12491249
],
12501250
)
1251+
@pytest.mark.parametrize("access_estimated_stepping_batches", [False, True])
12511252
def test_fit_loop_save_and_restore_dataloaders(
1252-
train_dataloader_factory, has_state, batches_before, batches_after, tmp_path
1253+
train_dataloader_factory, has_state, batches_before, batches_after, access_estimated_stepping_batches, tmp_path
12531254
):
12541255
"""Test that the CheckpointConnector saves the state of stateful dataloaders."""
12551256

@@ -1258,6 +1259,12 @@ def __init__(self):
12581259
super().__init__()
12591260
self.seen_data = []
12601261

1262+
def configure_optimizers(self):
1263+
if access_estimated_stepping_batches:
1264+
# sets up the dataloaders before the loop state gets restored from the checkpoint (#20550)
1265+
_ = int(self.trainer.estimated_stepping_batches)
1266+
return super().configure_optimizers()
1267+
12611268
def training_step(self, batch, batch_idx):
12621269
self.seen_data.append(batch)
12631270
print(batch)
@@ -1276,8 +1283,9 @@ def train_dataloader(self):
12761283
}
12771284

12781285
# Train for 2 steps
1286+
# `max_epochs` is set so that `estimated_stepping_batches` cannot take the infinite-training shortcut
12791287
model = DummyModel()
1280-
trainer = Trainer(**trainer_kwargs, max_steps=2)
1288+
trainer = Trainer(**trainer_kwargs, max_steps=2, max_epochs=10)
12811289
trainer.fit(model)
12821290
assert model.seen_data == batches_before
12831291

@@ -1291,6 +1299,6 @@ def train_dataloader(self):
12911299

12921300
# Restore training from step 2 and continue 2 more steps
12931301
model = DummyModel()
1294-
trainer = Trainer(**trainer_kwargs, max_steps=4)
1302+
trainer = Trainer(**trainer_kwargs, max_steps=4, max_epochs=10)
12951303
trainer.fit(model, ckpt_path=(tmp_path / "checkpoint.ckpt"))
12961304
assert model.seen_data == batches_after

0 commit comments

Comments
 (0)