Skip to content

Commit 7c362d7

Browse files
wanghan-iapcmHan Wang
andauthored
fix(jax): write the checkpoint pointer beside save_ckpt (#5726)
## Problem Fixes #5678. JAX training writes checkpoint directories and the stable `.jax` link relative to `save_ckpt` (which may include a directory), but always wrote the `checkpoint` pointer file to the current working directory with a value that still carried the directory prefix, e.g. `runs/water/model.ckpt.jax`. The freeze entrypoint looks for the pointer inside the folder it is given and resolves the pointer's value relative to that folder (`checkpoint_folder / pointer`). So for `save_ckpt = runs/water/model.ckpt`, the pointer was written to `./checkpoint` (not `runs/water/checkpoint`) and, even if relocated, its value would have double-prefixed to `runs/water/runs/water/model.ckpt.jax`. Passing `runs/water` to freeze or restart-style tooling could not find or resolve the checkpoint, even though the matching checkpoint directory and `.jax` link were written there. ## Fix Write the pointer into `Path(save_ckpt).parent` and store a value relative to that directory (the basename only). For the default bare `save_ckpt` (parent is `.`) the pointer stays in the CWD with the same value, so existing behavior is unchanged; only directory-valued `save_ckpt` is affected. ## Test Adds `source/tests/jax/test_checkpoint_pointer.py`, which drives `_save_checkpoint` with the checkpoint I/O mocked. The directory case asserts the pointer lands beside the checkpoint (`subdir/checkpoint`) with a basename value (`model.ckpt.jax`) and not in the CWD — this fails on master — and a bare-name control asserts the pointer stays in the CWD. The trainer's pointer writing previously had no coverage; the existing freeze test hand-wrote a correct pointer and never exercised the writer. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **Bug Fixes** - Fixed checkpoint “pointer” file placement to be written alongside the configured checkpoint save directory/path rather than always in the current working directory. - Updated pointer contents to reference the correct checkpoint name, improving downstream checkpoint resolution during freeze/restart. - **Tests** - Added unit tests validating pointer location and contents for both nested save paths and bare checkpoint filenames. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Han Wang <wang_han@iapcm.ac.cn>
1 parent d884779 commit 7c362d7

2 files changed

Lines changed: 73 additions & 2 deletions

File tree

deepmd/jax/train/trainer.py

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -841,8 +841,14 @@ def _save_checkpoint(self, step: int) -> None:
841841
log.info(f"Trained model has been saved to: {ckpt_path!s}")
842842
_link_checkpoint(ckpt_path, Path(f"{self.save_ckpt}.jax"))
843843
self._cleanup_old_checkpoints()
844-
with open("checkpoint", "w") as fp:
845-
fp.write(f"{self.save_ckpt}.jax")
844+
# Write the pointer next to the checkpoint prefix, with a value relative
845+
# to that directory (basename only). The freeze entrypoint looks for the
846+
# pointer inside the folder it is given and resolves the value relative
847+
# to it, so a directory-valued save_ckpt would otherwise be unresolvable.
848+
ckpt_dir = Path(self.save_ckpt).parent
849+
ckpt_dir.mkdir(parents=True, exist_ok=True)
850+
with open(ckpt_dir / "checkpoint", "w") as fp:
851+
fp.write(f"{Path(self.save_ckpt).name}.jax")
846852

847853
def _save_full_validation_checkpoint(
848854
self,
Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# SPDX-License-Identifier: LGPL-3.0-or-later
2+
"""Test the JAX trainer writes its checkpoint pointer beside save_ckpt.
3+
4+
JAX training writes checkpoint directories and the stable ``.jax`` link relative
5+
to ``save_ckpt`` (which may include a directory), but used to always write the
6+
``checkpoint`` pointer file to the current working directory with a value that
7+
still carried the directory prefix. The freeze entrypoint expects the pointer
8+
inside the folder it is given and resolves its value relative to that folder, so
9+
a directory-valued ``save_ckpt`` broke freeze/restart tooling.
10+
"""
11+
12+
import os
13+
import tempfile
14+
import unittest
15+
from pathlib import (
16+
Path,
17+
)
18+
from unittest.mock import (
19+
patch,
20+
)
21+
22+
from deepmd.jax.train.trainer import (
23+
DPTrainer,
24+
)
25+
26+
27+
class TestCheckpointPointer(unittest.TestCase):
28+
def setUp(self) -> None:
29+
self.tmpdir = tempfile.TemporaryDirectory()
30+
self.cwd = os.getcwd()
31+
os.chdir(self.tmpdir.name)
32+
33+
def tearDown(self) -> None:
34+
os.chdir(self.cwd)
35+
self.tmpdir.cleanup()
36+
37+
def _save(self, save_ckpt: str) -> None:
38+
trainer = DPTrainer.__new__(DPTrainer)
39+
trainer.save_ckpt = save_ckpt
40+
with (
41+
patch.object(DPTrainer, "_write_checkpoint"),
42+
patch.object(DPTrainer, "_cleanup_old_checkpoints"),
43+
patch("deepmd.jax.train.trainer._link_checkpoint"),
44+
):
45+
trainer._save_checkpoint(1)
46+
47+
def test_pointer_beside_ckpt_for_subdir(self) -> None:
48+
# save_ckpt with a directory: pointer must land in that directory with a
49+
# value relative to it (basename only), so freeze(subdir) resolves it.
50+
self._save("subdir/model.ckpt")
51+
pointer = Path("subdir") / "checkpoint"
52+
self.assertTrue(pointer.is_file())
53+
self.assertEqual(pointer.read_text(), "model.ckpt.jax")
54+
self.assertFalse(Path("checkpoint").exists())
55+
56+
def test_pointer_in_cwd_for_bare_name(self) -> None:
57+
# default/bare save_ckpt: pointer stays in the CWD (parent is ".").
58+
self._save("model.ckpt")
59+
pointer = Path("checkpoint")
60+
self.assertTrue(pointer.is_file())
61+
self.assertEqual(pointer.read_text(), "model.ckpt.jax")
62+
63+
64+
if __name__ == "__main__":
65+
unittest.main()

0 commit comments

Comments
 (0)