Skip to content

Commit 1ca7709

Browse files
committed
better support for local model registries, update test config
1 parent c7e0054 commit 1ca7709

4 files changed

Lines changed: 165 additions & 12 deletions

File tree

.github/workflows/ci.yml

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,13 @@ jobs:
8989
uses: actions/checkout@v4
9090
with:
9191
path: modflow-devtools
92-
92+
93+
- name: Checkout modflow6 for DFN autodiscovery
94+
uses: actions/checkout@v4
95+
with:
96+
repository: MODFLOW-ORG/modflow6
97+
path: modflow6
98+
9399
- name: Setup uv
94100
uses: astral-sh/setup-uv@v5
95101
with:
@@ -105,8 +111,9 @@ jobs:
105111
env:
106112
REPOS_PATH: ${{ github.workspace }}
107113
MODFLOW_DEVTOOLS_NO_AUTO_SYNC: 1
114+
TEST_DFN_PATH: ${{ github.workspace }}/modflow6/doc/mf6io/mf6ivar/dfn
108115
# use --dist loadfile to so tests requiring pytest-virtualenv run on the same worker
109-
run: uv run pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py --ignore test_models.py --ignore test_dfn_registry.py
116+
run: uv run pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py --ignore test_models.py
110117

111118
- name: Run network-dependent tests
112119
# only invoke the GH API on one OS and Python version
@@ -117,7 +124,8 @@ jobs:
117124
env:
118125
REPOS_PATH: ${{ github.workspace }}
119126
GITHUB_TOKEN: ${{ github.token }}
120-
# DFNs API
127+
# DFNs API - use test fork with registry file for RemoteDfnRegistry tests
128+
# Note: TEST_DFN_PATH is intentionally NOT set here to use fetch behavior
121129
TEST_DFNS_REPO: wpbonelli/modflow6
122130
TEST_DFNS_REF: registry
123131
TEST_DFNS_SOURCE: modflow6
@@ -130,7 +138,7 @@ jobs:
130138
TEST_PROGRAMS_REPO: MODFLOW-ORG/modflow6
131139
TEST_PROGRAMS_REF: develop
132140
TEST_PROGRAMS_SOURCE: modflow6
133-
run: uv run pytest -v -n auto --dist loadgroup --durations 0 test_download.py test_models.py test_dfn_registry.py
141+
run: uv run pytest -v -n auto --dist loadgroup --durations 0 test_download.py test_models.py test_dfns_registry.py
134142

135143
rtd:
136144
name: Docs

autotest/test_dfns_registry.py

Lines changed: 115 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,32 @@
2929
MF6_REPO = TEST_DFN_REPO.split("/")[1]
3030
MF6_REF = TEST_DFN_REF
3131

32+
# Path to cloned MF6 repository for autodiscovery (set by CI or local testing)
33+
# If set, use this instead of fetching individual DFN files
34+
TEST_DFN_PATH = os.getenv("TEST_DFN_PATH")
35+
3236

3337
@pytest.fixture(scope="module")
3438
def dfn_dir():
35-
"""Ensure DFN files are downloaded for testing."""
39+
"""
40+
Provide path to DFN files for testing.
41+
42+
Priority:
43+
1. If TEST_DFN_PATH is set, use the DFN directory from a cloned MF6 repo (autodiscovery)
44+
2. Otherwise, fetch individual DFN files to temp directory (legacy behavior)
45+
46+
The autodiscovery approach is preferred in CI to avoid needing registry files.
47+
"""
48+
# If TEST_DFN_PATH is set, use it (points to cloned MF6 DFN directory)
49+
if TEST_DFN_PATH:
50+
dfn_path = Path(TEST_DFN_PATH).expanduser().resolve()
51+
if not dfn_path.exists():
52+
raise ValueError(f"TEST_DFN_PATH={TEST_DFN_PATH} does not exist")
53+
if not any(dfn_path.glob("*.dfn")):
54+
raise ValueError(f"No DFN files found in TEST_DFN_PATH={TEST_DFN_PATH}")
55+
return dfn_path
56+
57+
# Fall back to fetching individual DFN files (legacy behavior for local development)
3658
if not any(DFN_DIR.glob("*.dfn")):
3759
fetch_dfns(MF6_OWNER, MF6_REPO, MF6_REF, DFN_DIR, verbose=True)
3860
return DFN_DIR
@@ -731,3 +753,95 @@ def test_get_sync_status(self):
731753
assert isinstance(status, dict)
732754
# All refs should be either True or False
733755
assert all(isinstance(v, bool) for v in status.values())
756+
757+
758+
@requires_pkg("boltons", "pydantic")
759+
class TestGetRegistryWithPath:
760+
"""Tests for get_registry() with path parameter."""
761+
762+
def test_get_registry_with_path_returns_local_registry(self, dfn_dir):
763+
"""Test that get_registry with path returns LocalDfnRegistry."""
764+
from modflow_devtools.dfns.registry import LocalDfnRegistry, get_registry
765+
766+
registry = get_registry(path=dfn_dir)
767+
768+
assert isinstance(registry, LocalDfnRegistry)
769+
assert registry.path == dfn_dir.resolve()
770+
771+
def test_get_registry_with_path_and_metadata(self, dfn_dir):
772+
"""Test that source/ref metadata is preserved with path."""
773+
from modflow_devtools.dfns.registry import get_registry
774+
775+
registry = get_registry(path=dfn_dir, source="test", ref="local")
776+
777+
assert registry.source == "test"
778+
assert registry.ref == "local"
779+
780+
def test_get_registry_without_path_returns_remote_registry(self):
781+
"""Test that get_registry without path still returns RemoteDfnRegistry."""
782+
from modflow_devtools.dfns.registry import RemoteDfnRegistry, get_registry
783+
784+
registry = get_registry(source="modflow6", ref="develop", auto_sync=False)
785+
786+
assert isinstance(registry, RemoteDfnRegistry)
787+
788+
789+
@requires_pkg("boltons", "pydantic")
790+
class TestConvenienceFunctionsWithPath:
791+
"""Tests for convenience functions with path parameter."""
792+
793+
def test_get_dfn_with_path(self, dfn_dir):
794+
"""Test get_dfn() with path parameter."""
795+
from modflow_devtools.dfns import get_dfn
796+
797+
dfn = get_dfn("gwf-chd", path=dfn_dir)
798+
799+
assert dfn.name == "gwf-chd"
800+
assert dfn.parent == "gwf-nam"
801+
802+
def test_get_dfn_path_with_path(self, dfn_dir):
803+
"""Test get_dfn_path() with path parameter."""
804+
from modflow_devtools.dfns import get_dfn_path
805+
806+
file_path = get_dfn_path("gwf-chd", path=dfn_dir)
807+
808+
assert file_path.exists()
809+
assert file_path.name == "gwf-chd.dfn"
810+
811+
def test_list_components_with_path(self, dfn_dir):
812+
"""Test list_components() with path parameter."""
813+
from modflow_devtools.dfns import list_components
814+
815+
components = list_components(path=dfn_dir)
816+
817+
assert len(components) > 100
818+
assert "gwf-chd" in components
819+
820+
821+
@requires_pkg("boltons", "pydantic")
822+
def test_autodiscovery_workflow(dfn_dir):
823+
"""Test complete autodiscovery workflow."""
824+
from modflow_devtools.dfns import get_dfn, get_registry, list_components
825+
826+
# Get registry pointing at local directory
827+
registry = get_registry(path=dfn_dir, ref="local")
828+
829+
# List components
830+
components = list(registry.spec.keys())
831+
assert len(components) > 100
832+
833+
# Get specific DFN
834+
gwf_chd = registry.get_dfn("gwf-chd")
835+
assert gwf_chd.name == "gwf-chd"
836+
assert gwf_chd.blocks is not None
837+
838+
# Get file path
839+
chd_path = registry.get_dfn_path("gwf-chd")
840+
assert chd_path.exists()
841+
842+
# Use convenience functions
843+
components_list = list_components(path=dfn_dir)
844+
assert "gwf-chd" in components_list
845+
846+
dfn = get_dfn("gwf-wel", path=dfn_dir)
847+
assert dfn.name == "gwf-wel"

modflow_devtools/dfns/__init__.py

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,7 @@ def get_dfn(
794794
component: str,
795795
ref: str = "develop",
796796
source: str = "modflow6",
797+
path: str | PathLike | None = None,
797798
) -> "Dfn":
798799
"""
799800
Get a DFN by component name from the registry.
@@ -809,6 +810,9 @@ def get_dfn(
809810
Git ref (branch, tag, or commit hash). Default is "develop".
810811
source : str, optional
811812
Source repository name. Default is "modflow6".
813+
path : str or PathLike, optional
814+
Path to a local directory containing DFN files. If provided,
815+
uses autodiscovery from local filesystem instead of remote.
812816
813817
Returns
814818
-------
@@ -819,16 +823,18 @@ def get_dfn(
819823
--------
820824
>>> dfn = get_dfn("gwf-chd")
821825
>>> dfn = get_dfn("gwf-chd", ref="6.6.0")
826+
>>> dfn = get_dfn("gwf-chd", path="/path/to/dfns")
822827
"""
823828
registry = _get_registry_module()
824-
reg = registry.get_registry(source=source, ref=ref)
829+
reg = registry.get_registry(source=source, ref=ref, path=path)
825830
return reg.get_dfn(component)
826831

827832

828833
def get_dfn_path(
829834
component: str,
830835
ref: str = "develop",
831836
source: str = "modflow6",
837+
path: str | PathLike | None = None,
832838
) -> Path:
833839
"""
834840
Get the local cached file path for a DFN component.
@@ -841,24 +847,29 @@ def get_dfn_path(
841847
Git ref (branch, tag, or commit hash). Default is "develop".
842848
source : str, optional
843849
Source repository name. Default is "modflow6".
850+
path : str or PathLike, optional
851+
Path to a local directory containing DFN files. If provided,
852+
returns path from local filesystem instead of cache.
844853
845854
Returns
846855
-------
847856
Path
848-
Path to the local cached DFN file.
857+
Path to the local DFN file (cached or local directory).
849858
850859
Examples
851860
--------
852861
>>> path = get_dfn_path("gwf-chd", ref="6.6.0")
862+
>>> path = get_dfn_path("gwf-chd", path="/path/to/dfns")
853863
"""
854864
registry = _get_registry_module()
855-
reg = registry.get_registry(source=source, ref=ref)
865+
reg = registry.get_registry(source=source, ref=ref, path=path)
856866
return reg.get_dfn_path(component)
857867

858868

859869
def list_components(
860870
ref: str = "develop",
861871
source: str = "modflow6",
872+
path: str | PathLike | None = None,
862873
) -> list[str]:
863874
"""
864875
List available components for a registry.
@@ -869,6 +880,9 @@ def list_components(
869880
Git ref (branch, tag, or commit hash). Default is "develop".
870881
source : str, optional
871882
Source repository name. Default is "modflow6".
883+
path : str or PathLike, optional
884+
Path to a local directory containing DFN files. If provided,
885+
lists components from local filesystem.
872886
873887
Returns
874888
-------
@@ -880,7 +894,8 @@ def list_components(
880894
>>> components = list_components(ref="6.6.0")
881895
>>> "gwf-chd" in components
882896
True
897+
>>> components = list_components(path="/path/to/dfns")
883898
"""
884899
registry = _get_registry_module()
885-
reg = registry.get_registry(source=source, ref=ref)
900+
reg = registry.get_registry(source=source, ref=ref, path=path)
886901
return list(reg.spec.keys())

modflow_devtools/dfns/registry.py

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -735,7 +735,8 @@ def get_registry(
735735
source: str = "modflow6",
736736
ref: str = "develop",
737737
auto_sync: bool = True,
738-
) -> RemoteDfnRegistry:
738+
path: str | PathLike | None = None,
739+
) -> DfnRegistry:
739740
"""
740741
Get a registry for the specified source and ref.
741742
@@ -748,17 +749,32 @@ def get_registry(
748749
auto_sync : bool, optional
749750
If True and registry is not cached, automatically sync. Default is True.
750751
Can be disabled via MODFLOW_DEVTOOLS_NO_AUTO_SYNC environment variable.
752+
Ignored when path is provided.
753+
path : str or PathLike, optional
754+
Path to a local directory containing DFN files. If provided, returns
755+
a LocalDfnRegistry for autodiscovery instead of RemoteDfnRegistry.
756+
When using a local path, source and ref are used for metadata only.
751757
752758
Returns
753759
-------
754-
RemoteDfnRegistry
755-
Registry for the specified source and ref.
760+
DfnRegistry
761+
Registry for the specified source and ref. Returns LocalDfnRegistry
762+
if path is provided, otherwise RemoteDfnRegistry.
756763
757764
Examples
758765
--------
766+
>>> # Remote registry (existing behavior)
759767
>>> registry = get_registry(ref="6.6.0")
760768
>>> dfn = registry.get_dfn("gwf-chd")
769+
770+
>>> # Local registry with autodiscovery (NEW)
771+
>>> registry = get_registry(path="/path/to/mf6/doc/mf6io/mf6ivar/dfn")
772+
>>> dfn = registry.get_dfn("gwf-chd")
761773
"""
774+
# If path is provided, return LocalDfnRegistry for autodiscovery
775+
if path is not None:
776+
return LocalDfnRegistry(path=path, source=source, ref=ref)
777+
762778
# Check for auto-sync opt-out
763779
if os.environ.get("MODFLOW_DEVTOOLS_NO_AUTO_SYNC", "").lower() in ("1", "true", "yes"):
764780
auto_sync = False

0 commit comments

Comments
 (0)