diff --git a/source/lmp/tests/model_convert.py b/source/lmp/tests/model_convert.py new file mode 100644 index 0000000000..18047f3707 --- /dev/null +++ b/source/lmp/tests/model_convert.py @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: LGPL-3.0-or-later +"""Helpers for preparing converted TensorFlow graph files in LAMMPS tests.""" + +from __future__ import ( + annotations, +) + +import errno +import os +import subprocess as sp +import sys +import tempfile +import time +from pathlib import ( + Path, +) + +_LOCK_TIMEOUT_SECONDS = 60.0 +_LOCK_POLL_SECONDS = 0.1 + + +def _is_up_to_date(source: Path, output: Path) -> bool: + return output.exists() and output.stat().st_mtime_ns >= source.stat().st_mtime_ns + + +def _read_lock_pid(lock_file: Path) -> int | None: + try: + for line in lock_file.read_text(encoding="utf-8").splitlines(): + if line.startswith("pid="): + return int(line.split("=", maxsplit=1)[1]) + except (FileNotFoundError, ValueError): + return None + return None + + +def _pid_is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError as err: + if err.errno == errno.ESRCH: + return False + raise + return True + + +def _should_break_stale_lock(lock_file: Path) -> bool: + try: + lock_stat = lock_file.stat() + except FileNotFoundError: + return False + + lock_pid = _read_lock_pid(lock_file) + if lock_pid is not None: + return not _pid_is_running(lock_pid) + + lock_age = time.time() - lock_stat.st_mtime + return lock_age > _LOCK_TIMEOUT_SECONDS + + +def ensure_converted_pb(source: Path, output: Path) -> Path: + """Convert ``source`` into ``output`` only when the target is missing or stale. + + The conversion is protected by a simple lock file and uses atomic replacement so + repeated imports across multiple test modules do not regenerate the same model + more than once. + """ + source = source.resolve() + output = output.resolve() + output.parent.mkdir(parents=True, exist_ok=True) + lock_file = output.with_name(f".{output.name}.lock") + started = time.monotonic() + + while True: + if _is_up_to_date(source, output): + return output + try: + fd = os.open(str(lock_file), os.O_CREAT | os.O_EXCL | os.O_WRONLY) + except FileExistsError as err: + if _should_break_stale_lock(lock_file): + lock_file.unlink(missing_ok=True) + continue + if time.monotonic() - started >= _LOCK_TIMEOUT_SECONDS: + raise TimeoutError(f"Timed out waiting for {lock_file}") from err + time.sleep(_LOCK_POLL_SECONDS) + continue + break + + tmp_path: Path | None = None + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(f"pid={os.getpid()}\n") + + if _is_up_to_date(source, output): + return output + + tmp_fd, tmp_name = tempfile.mkstemp( + dir=output.parent, + prefix=f".{output.name}.", + ) + os.close(tmp_fd) + tmp_path = Path(tmp_name) + sp.run( + [ + sys.executable, + "-m", + "deepmd", + "convert-from", + "pbtxt", + "-i", + str(source), + "-o", + str(tmp_path), + ], + check=True, + ) + tmp_path.replace(output) + tmp_path = None + return output + finally: + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + lock_file.unlink(missing_ok=True) diff --git a/source/lmp/tests/test_deeptensor.py b/source/lmp/tests/test_deeptensor.py index 20be3033b8..c0f9a2d7f7 100644 --- a/source/lmp/tests/test_deeptensor.py +++ b/source/lmp/tests/test_deeptensor.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: LGPL-3.0-or-later import os -import subprocess as sp -import sys from pathlib import ( Path, ) @@ -12,6 +10,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -56,20 +57,14 @@ # type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) - -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) # TODO # write_lmp_data(box, coord, type_HO, data_type_map_file) diff --git a/source/lmp/tests/test_dplr.py b/source/lmp/tests/test_dplr.py index dd0c03aabe..defdff21c0 100644 --- a/source/lmp/tests/test_dplr.py +++ b/source/lmp/tests/test_dplr.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: LGPL-3.0-or-later import os -import subprocess as sp -import sys from pathlib import ( Path, ) @@ -12,6 +10,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data_full, ) @@ -265,16 +266,14 @@ mesh = 10 -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + ensure_converted_pb(dipole_pbtxt_file, dipole_pb_file) + write_lmp_data_full( box, coord, mol_list, type_OH, charge, data_file, bond_list, mass_list ) diff --git a/source/lmp/tests/test_lammps.py b/source/lmp/tests/test_lammps.py index 0e53b38d7b..eca782365e 100644 --- a/source/lmp/tests/test_lammps.py +++ b/source/lmp/tests/test_lammps.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -221,19 +224,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_3types.py b/source/lmp/tests/test_lammps_3types.py index 9156914dbc..a3cef29a62 100644 --- a/source/lmp/tests/test_lammps_3types.py +++ b/source/lmp/tests/test_lammps_3types.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: LGPL-3.0-or-later import os -import subprocess as sp -import sys from pathlib import ( Path, ) @@ -11,6 +9,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -244,19 +245,15 @@ # https://github.com/lammps/lammps/blob/1e1311cf401c5fc2614b5d6d0ff3230642b76597/src/update.cpp#L193 nktv2p = 1.6021765e6 -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) diff --git a/source/lmp/tests/test_lammps_dpa_jax.py b/source/lmp/tests/test_lammps_dpa_jax.py index 51b2d56742..d4d63dd340 100644 --- a/source/lmp/tests/test_lammps_dpa_jax.py +++ b/source/lmp/tests/test_lammps_dpa_jax.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -222,16 +225,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module(): if os.environ.get("ENABLE_JAX", "1") != "1": pytest.skip( "Skip test because JAX support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_dpa_pt.py b/source/lmp/tests/test_lammps_dpa_pt.py index 6dfa6099e2..2c93096556 100644 --- a/source/lmp/tests/test_lammps_dpa_pt.py +++ b/source/lmp/tests/test_lammps_dpa_pt.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -220,16 +223,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_PYTORCH", "1") != "1": pytest.skip( "Skip test because PyTorch support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_dpa_pt_nopbc.py b/source/lmp/tests/test_lammps_dpa_pt_nopbc.py index 989a782b5f..0edd3ed089 100644 --- a/source/lmp/tests/test_lammps_dpa_pt_nopbc.py +++ b/source/lmp/tests/test_lammps_dpa_pt_nopbc.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -218,16 +221,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_PYTORCH", "1") != "1": pytest.skip( "Skip test because PyTorch support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_dpa_sel_pt.py b/source/lmp/tests/test_lammps_dpa_sel_pt.py index f65c710409..27baee3432 100644 --- a/source/lmp/tests/test_lammps_dpa_sel_pt.py +++ b/source/lmp/tests/test_lammps_dpa_sel_pt.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -223,16 +226,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_PYTORCH", "1") != "1": pytest.skip( "Skip test because PyTorch support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( @@ -721,6 +722,10 @@ def test_pair_deepmd_si(lammps_si) -> None: ("balance_args",), [(["--balance"],), ([],)], ) +@pytest.mark.skipif( + os.environ.get("ENABLE_TENSORFLOW", "1") != "1", + reason="Skip test because TensorFlow support is not enabled.", +) def test_pair_deepmd_mpi(balance_args: list) -> None: with tempfile.NamedTemporaryFile() as f: sp.check_call( diff --git a/source/lmp/tests/test_lammps_faparam.py b/source/lmp/tests/test_lammps_faparam.py index 1a614c3d24..c40fb9bdb0 100644 --- a/source/lmp/tests/test_lammps_faparam.py +++ b/source/lmp/tests/test_lammps_faparam.py @@ -2,8 +2,6 @@ """Test LAMMPS fparam and aparam input.""" import os -import subprocess as sp -import sys from pathlib import ( Path, ) @@ -14,6 +12,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -134,16 +135,13 @@ type_OH = np.array([1, 1, 1, 1, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + write_lmp_data(box, coord, type_OH, data_file) diff --git a/source/lmp/tests/test_lammps_jax.py b/source/lmp/tests/test_lammps_jax.py index e3d0e5ce74..dcc20a1e85 100644 --- a/source/lmp/tests/test_lammps_jax.py +++ b/source/lmp/tests/test_lammps_jax.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -222,16 +225,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module(): if os.environ.get("ENABLE_JAX", "1") != "1": pytest.skip( "Skip test because JAX support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_pd.py b/source/lmp/tests/test_lammps_pd.py index 85275c4027..2c83a65b80 100644 --- a/source/lmp/tests/test_lammps_pd.py +++ b/source/lmp/tests/test_lammps_pd.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -223,16 +226,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module(): if os.environ.get("ENABLE_PADDLE", "1") != "1": pytest.skip( "Skip test because Paddle support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_pt.py b/source/lmp/tests/test_lammps_pt.py index f6fb8f949b..a95cc4dbaa 100644 --- a/source/lmp/tests/test_lammps_pt.py +++ b/source/lmp/tests/test_lammps_pt.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data, ) @@ -220,16 +223,14 @@ type_HO = np.array([2, 1, 1, 2, 1, 1]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_PYTORCH", "1") != "1": pytest.skip( "Skip test because PyTorch support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data(box, coord, type_OH, data_file) write_lmp_data(box, coord, type_HO, data_type_map_file) write_lmp_data( diff --git a/source/lmp/tests/test_lammps_spin.py b/source/lmp/tests/test_lammps_spin.py index 79050297a5..a5210e2390 100644 --- a/source/lmp/tests/test_lammps_spin.py +++ b/source/lmp/tests/test_lammps_spin.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data_spin, ) @@ -210,19 +213,14 @@ type_NiO = np.array([1, 1, 2, 2]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data_spin(box, coord, spin, type_NiO, data_file) diff --git a/source/lmp/tests/test_lammps_spin_nopbc.py b/source/lmp/tests/test_lammps_spin_nopbc.py index 718a8e2ccc..309d1591cc 100644 --- a/source/lmp/tests/test_lammps_spin_nopbc.py +++ b/source/lmp/tests/test_lammps_spin_nopbc.py @@ -14,6 +14,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data_spin, ) @@ -208,19 +211,15 @@ ) type_NiO = np.array([1, 1, 2, 2]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file.resolve()} -o {pb_file.resolve()}".split() -) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - def setup_module() -> None: if os.environ.get("ENABLE_TENSORFLOW", "1") != "1": pytest.skip( "Skip test because TensorFlow support is not enabled.", ) + ensure_converted_pb(pbtxt_file, pb_file) + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data_spin(box, coord, spin, type_NiO, data_file) diff --git a/source/lmp/tests/test_lammps_spin_nopbc_pt.py b/source/lmp/tests/test_lammps_spin_nopbc_pt.py index 486538fe8b..86d74f68c4 100644 --- a/source/lmp/tests/test_lammps_spin_nopbc_pt.py +++ b/source/lmp/tests/test_lammps_spin_nopbc_pt.py @@ -14,6 +14,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data_spin, ) @@ -87,16 +90,14 @@ type_NiO = np.array([1, 1, 2, 2]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_PYTORCH", "1") != "1": pytest.skip( "Skip test because PyTorch support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data_spin(box, coord, spin, type_NiO, data_file) diff --git a/source/lmp/tests/test_lammps_spin_pt.py b/source/lmp/tests/test_lammps_spin_pt.py index 8a21f27710..ff9fc6ecb8 100644 --- a/source/lmp/tests/test_lammps_spin_pt.py +++ b/source/lmp/tests/test_lammps_spin_pt.py @@ -15,6 +15,9 @@ from lammps import ( PyLammps, ) +from model_convert import ( + ensure_converted_pb, +) from write_lmp_data import ( write_lmp_data_spin, ) @@ -178,16 +181,14 @@ type_NiO = np.array([1, 1, 2, 2]) -sp.check_output( - f"{sys.executable} -m deepmd convert-from pbtxt -i {pbtxt_file2.resolve()} -o {pb_file2.resolve()}".split() -) - - def setup_module() -> None: if os.environ.get("ENABLE_PYTORCH", "1") != "1": pytest.skip( "Skip test because PyTorch support is not enabled.", ) + if os.environ.get("ENABLE_TENSORFLOW", "1") == "1": + ensure_converted_pb(pbtxt_file2, pb_file2) + write_lmp_data_spin(box, coord, spin, type_NiO, data_file)