feat(dfns): specify and reimplement v2 dfn schema (#311)

Draft a new DFN specification and schema in the dfns module. Simplify the DFNs API. Clean up the older dfn module. Rewrite and reorganize some docs. Drop Python3.10 and add Python3.14.

The schema mapping is messy, but temporary. Once we move to v2 upstream in mf6 it can go away. The tentative v2 spec/schema is what really matters here.

Author: wpbonelli

.github/workflows/ci.yml

@@ -51,10 +51,10 @@ jobs:
     steps:
 
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           cache-dependency-glob: "**/pyproject.toml"
 
@@ -80,9 +80,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-22.04, macos-14, windows-2022 ]
-        python: [ "3.10", "3.11", "3.12", "3.13" ]
+        python: [ "3.11", "3.12", "3.13", "3.14" ]
     env:
       GCC_V: 11
+      UV_PROJECT_ENVIRONMENT: ${{ github.workspace }}/.venv
     steps:
 
       - name: Checkout repo
@@ -97,10 +98,11 @@ jobs:
           path: modflow6
 
       - name: Setup uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
           cache-dependency-glob: "**/pyproject.toml"
           python-version: ${{ matrix.python }}
+          working-directory: modflow-devtools
 
       - name: Setup Fortran
         if: runner.os != 'Windows'
@@ -124,16 +126,16 @@ jobs:
         working-directory: modflow-devtools/autotest
         env:
           REPOS_PATH: ${{ github.workspace }}
+          DFNS_PATH: ${{ github.workspace }}/modflow6/doc/mf6io/mf6ivar/dfn
           MODFLOW_DEVTOOLS_AUTO_SYNC: 0
-          TEST_DFN_PATH: ${{ github.workspace }}/modflow6/doc/mf6io/mf6ivar/dfn
         # use --dist loadfile to so tests requiring pytest-virtualenv run on the same worker
-        run: uv run pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py --ignore test_models.py --ignore test_dfns_registry.py
+        run: uv run pytest -v -n auto --dist loadfile --durations 0 --ignore test_download.py --ignore test_models.py --ignore dfns/test_dfns_registry.py
 
       - name: Run network-dependent tests
         # only invoke the GH API on one OS and Python version
         # to avoid rate limits (1000 rqs / hour / repository)
         # https://docs.github.com/en/actions/learn-github-actions/usage-limits-billing-and-administration#usage-limits
-        if: runner.os == 'Linux' && matrix.python == '3.10'
+        if: runner.os == 'Linux' && matrix.python == '3.11'
         working-directory: modflow-devtools/autotest
         env:
           REPOS_PATH: ${{ github.workspace }}
@@ -151,7 +153,7 @@ jobs:
           TEST_PROGRAMS_REPO: MODFLOW-ORG/modflow6
           TEST_PROGRAMS_REF: develop
           TEST_PROGRAMS_SOURCE: modflow6
-        run: uv run pytest -v -n auto --dist loadgroup --durations 0 test_download.py test_models.py test_dfns_registry.py
+        run: uv run pytest -v -n auto --dist loadgroup --durations 0 test_download.py test_models.py dfns/test_dfns_registry.py
 
   rtd:
     name: Docs

DEVELOPER.md

@@ -19,7 +19,7 @@ This document provides guidance to set up a development environment and discusse
 
 ## Requirements
 
-Python3.10+ is currently required. This project has historically aimed to support several recent versions of Python, loosely following [NEP 29](https://numpy.org/neps/nep-0029-deprecation_policy.html#implementation). In current and future development this window may narrow to follow [SPEC 0](https://scientific-python.org/specs/spec-0000/#support-window) instead.
+Python3.11+. This project has historically aimed to support several recent versions of Python, loosely following [NEP 29](https://numpy.org/neps/nep-0029-deprecation_policy.html#implementation). In current and future development this window may narrow to follow [SPEC 0](https://scientific-python.org/specs/spec-0000/#support-window) instead.
 
 ## Installation
 

README.md

@@ -41,7 +41,7 @@ Python development tools for MODFLOW 6 and related projects.
 
 ## Requirements
 
-Python3.10+, dependency-free by default.
+Python3.11+, dependency-free by default.
 
 Two main dependency groups are available, oriented around specific use cases:
 

autotest/dfn/__init__.py

@@ -0,0 +1,16 @@
+import pytest
+
+from modflow_devtools.dfn import fetch_dfns
+
+MF6_OWNER = "MODFLOW-ORG"
+MF6_REPO = "modflow6"
+MF6_REF = "develop"
+
+
+@pytest.fixture(scope="module")
+def dfn_dir(module_tmpdir):
+    pytest.importorskip("boltons")
+    path = module_tmpdir / "dfn"
+    path.mkdir()
+    fetch_dfns(MF6_OWNER, MF6_REPO, MF6_REF, path, verbose=True)
+    return path

autotest/dfn/conftest.py

@@ -0,0 +1,23 @@
+from modflow_devtools.dfn import Dfn
+from modflow_devtools.markers import requires_pkg
+
+
+@requires_pkg("boltons")
+def test_load_v1(dfn_dir):
+    common = {}
+    common_path = dfn_dir / "common.dfn"
+    if common_path.exists():
+        with common_path.open() as f:
+            common, _ = Dfn._load_v1_flat(f)
+    names = [p.stem for p in dfn_dir.glob("*.dfn") if p.stem not in ("common", "flopy")]
+    assert names
+    for name in names:
+        with (dfn_dir / f"{name}.dfn").open() as f:
+            dfn = Dfn.load(f, name=name, common=common)
+        assert any(dfn)
+
+
+@requires_pkg("boltons")
+def test_load_all(dfn_dir):
+    dfns = Dfn.load_all(dfn_dir)
+    assert any(dfns)

autotest/dfn/test_dfn.py

@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+
+import pytest
+
+from modflow_devtools.dfns import fetch_dfns
+
+PROJ_ROOT = Path(__file__).parents[1]
+
+DFNS_REPO = os.getenv("TEST_DFNS_REPO", "MODFLOW-ORG/modflow6")
+DFNS_REF = os.getenv("TEST_DFNS_REF", "develop")
+DFNS_SOURCE = os.getenv("TEST_DFNS_SOURCE", "modflow6")
+DFNS_VERSION = os.getenv("TEST_DFNS_VERSION", "6.6.0")
+
+
+@pytest.fixture(scope="module")
+def dfn_dir(module_tmpdir):
+    """
+    Path to DFN files: $DFNS_PATH if set, otherwise fetched from develop branch
+    to a temp dir (for LocalDfnRegistry tests).
+    """
+    env_var = "DFNS_PATH"
+    if dfns_path := os.getenv(env_var):
+        dfn_path = Path(dfns_path).expanduser().resolve()
+        assert dfn_path.exists(), f"{env_var}={dfns_path} does not exist"
+        assert any(dfn_path.glob("*.dfn")), f"{env_var}={dfns_path} empty"
+        return dfn_path
+
+    dfns_path = module_tmpdir / "dfns"
+    dfns_path.mkdir()
+    owner = DFNS_REPO.split("/")[0]
+    repo = DFNS_REPO.split("/")[1]
+    fetch_dfns(owner, repo, DFNS_REF, dfns_path, verbose=True)
+    return dfns_path

autotest/dfns/__init__.py

@@ -0,0 +1,56 @@
+from modflow_devtools.dfns import Dfns
+from modflow_devtools.markers import requires_pkg
+
+
+def test_load(dfn_dir):
+    spec = Dfns.load(dfn_dir)
+    assert spec.schema_version == "2"
+    assert spec.root is not None
+    assert spec.root.name == "sim-nam"
+    assert len(spec.components) > 100
+    assert "sim-nam" in spec.components
+    assert "gwf-nam" in spec.components
+    assert "gwf-chd" in spec.components
+    assert "gwf-wel" in spec.components.keys()
+    assert "garbage" not in spec.components
+
+    gwf_chd = spec.components["gwf-chd"]
+    assert gwf_chd.name == "gwf-chd"
+    assert gwf_chd.parent == "gwf-nam"
+
+    sim_children = spec.children("sim-nam")
+    assert "gwf-nam" in sim_children
+
+    gwf_children = spec.children("gwf-nam")
+    assert "gwf-chd" in gwf_children
+
+
+def test_load_empty_directory(function_tmpdir):
+    spec = Dfns.load(function_tmpdir)
+    assert len(spec.components) == 0
+
+
+# =============================================================================
+# CLI
+# =============================================================================
+
+
+@requires_pkg("pydantic")
+class TestCLI:
+    def test_main_help(self):
+        from modflow_devtools.dfns.__main__ import main
+
+        result = main([])
+        assert result == 0
+
+    def test_info_command(self):
+        from modflow_devtools.dfns.__main__ import main
+
+        result = main(["info"])
+        assert result == 0
+
+    def test_clean_command(self):
+        from modflow_devtools.dfns.__main__ import main
+
+        result = main(["clean"])
+        assert result == 0