fix(dfn): restore old dfn2toml logic (#299)

Forgot to split out a separate copy of the TOML conversion script in #271. This makes 1.9.0 breaking, unfortunately. Will need to make a patch release and exclude 1.9.0 from downstream dependencies.

Two versions of this script can live side by side for now, until the modflow_devtools.dfns module is properly released with 2.x.

Author: wpbonelli

autotest/test_dfns.py

@@ -4,15 +4,15 @@
 import pytest
 from packaging.version import Version
 
-from modflow_devtools.dfn2toml import convert, is_valid
 from modflow_devtools.dfns import Dfn, _load_common, load, load_flat
+from modflow_devtools.dfns.dfn2toml import convert, is_valid
 from modflow_devtools.dfns.fetch import fetch_dfns
 from modflow_devtools.dfns.schema.v1 import FieldV1
 from modflow_devtools.dfns.schema.v2 import FieldV2
 from modflow_devtools.markers import requires_pkg
 
 PROJ_ROOT = Path(__file__).parents[1]
-DFN_DIR = PROJ_ROOT / "autotest" / "temp" / "dfn"
+DFN_DIR = PROJ_ROOT / "autotest" / "temp" / "dfns"
 TOML_DIR = DFN_DIR / "toml"
 SPEC_DIRS = {1: DFN_DIR, 2: TOML_DIR}
 MF6_OWNER = "MODFLOW-ORG"

modflow_devtools/dfn2toml.py

@@ -1,126 +1,46 @@
 """Convert DFNs to TOML."""
 
 import argparse
-import sys
-import textwrap
-from dataclasses import asdict
 from os import PathLike
 from pathlib import Path
 
 import tomli_w as tomli
 from boltons.iterutils import remap
 
-from modflow_devtools.dfns import Dfn, is_valid, load, load_flat, map, to_flat, to_tree
-from modflow_devtools.dfns.schema.block import block_sort_key
-from modflow_devtools.misc import drop_none_or_empty
+from modflow_devtools.dfn import Dfn
 
 # mypy: ignore-errors
 
 
-def convert(inpath: PathLike, outdir: PathLike, schema_version: str = "2") -> None:
-    """
-    Convert DFN files in `inpath` to TOML files in `outdir`.
-    By default, convert the definitions to schema version 2.
-    """
-    inpath = Path(inpath).expanduser().absolute()
+def convert(indir: PathLike, outdir: PathLike):
+    indir = Path(indir).expanduser().absolute()
     outdir = Path(outdir).expanduser().absolute()
     outdir.mkdir(exist_ok=True, parents=True)
+    for dfn in Dfn.load_all(indir).values():
+        with Path.open(outdir / f"{dfn['name']}.toml", "wb") as f:
 
-    if inpath.is_file():
-        if inpath.name == "common.dfn":
-            raise ValueError("Cannot convert common.dfn as a standalone file")
+            def drop_none_or_empty(path, key, value):
+                if value is None or value == "" or value == [] or value == {}:
+                    return False
+                return True
 
-        common_path = inpath.parent / "common.dfn"
-        if common_path.exists():
-            with common_path.open() as f:
-                from modflow_devtools.dfn import parse_dfn
-
-                common, _ = parse_dfn(f)
-        else:
-            common = {}
-
-        with inpath.open() as f:
-            dfn = load(f, name=inpath.stem, common=common, format="dfn")
-
-        dfn = map(dfn, schema_version=schema_version)
-        _convert(dfn, outdir / f"{inpath.stem}.toml")
-    else:
-        dfns = {
-            name: map(dfn, schema_version=schema_version) for name, dfn in load_flat(inpath).items()
-        }
-        tree = to_tree(dfns)
-        flat = to_flat(tree)
-        for dfn_name, dfn in flat.items():
-            _convert(dfn, outdir / f"{dfn_name}.toml")
-
-
-def _convert(dfn: Dfn, outpath: Path) -> None:
-    with Path.open(outpath, "wb") as f:
-        # TODO if we start using c/attrs, swap out
-        # all this for a custom unstructuring hook
-        dfn_dict = asdict(dfn)
-        dfn_dict["schema_version"] = str(dfn_dict["schema_version"])
-        if blocks := dfn_dict.pop("blocks", None):
-            for block_name, block_fields in blocks.items():
-                if block_name not in dfn_dict:
-                    dfn_dict[block_name] = {}
-                for field_name, field_data in block_fields.items():
-                    dfn_dict[block_name][field_name] = field_data
-
-        tomli.dump(
-            dict(
-                sorted(
-                    remap(dfn_dict, visit=drop_none_or_empty).items(),
-                    key=block_sort_key,
-                )
-            ),
-            f,
-        )
+            tomli.dump(remap(dfn, visit=drop_none_or_empty), f)
 
 
 if __name__ == "__main__":
-    """
-    Convert DFN files in the original format and schema version 1
-    to TOML files, by default also converting to schema version 2.
-    """
+    """Convert DFN files to TOML."""
 
-    parser = argparse.ArgumentParser(
-        description="Convert DFN files to TOML.",
-        epilog=textwrap.dedent(
-            """\
-Convert DFN files in the original format and schema version 1
-to TOML files, by default also converting to schema version 2.
-"""
-        ),
-    )
+    parser = argparse.ArgumentParser(description="Convert DFN files to TOML.")
     parser.add_argument(
         "--indir",
         "-i",
         type=str,
-        help="Directory containing DFN files, or a single DFN file.",
+        help="Directory containing DFN files.",
     )
     parser.add_argument(
         "--outdir",
         "-o",
         help="Output directory.",
     )
-    parser.add_argument(
-        "--schema-version",
-        "-s",
-        type=str,
-        default="2",
-        help="Schema version to convert to.",
-    )
-    parser.add_argument(
-        "--validate",
-        "-v",
-        action="store_true",
-        help="Validate DFN files without converting them.",
-    )
     args = parser.parse_args()
-
-    if args.validate:
-        if not is_valid(args.indir):
-            sys.exit(1)
-    else:
-        convert(args.indir, args.outdir, args.schema_version)
+    convert(args.indir, args.outdir)

modflow_devtools/dfns/dfn2toml.py

@@ -0,0 +1,125 @@
+"""Convert DFNs to TOML."""
+
+import argparse
+import sys
+import textwrap
+from dataclasses import asdict
+from os import PathLike
+from pathlib import Path
+
+import tomli_w as tomli
+from boltons.iterutils import remap
+
+from modflow_devtools.dfns import Dfn, is_valid, load, load_flat, map, to_flat, to_tree
+from modflow_devtools.dfns.parse import parse_dfn
+from modflow_devtools.dfns.schema.block import block_sort_key
+from modflow_devtools.misc import drop_none_or_empty
+
+# mypy: ignore-errors
+
+
+def convert(inpath: PathLike, outdir: PathLike, schema_version: str = "2") -> None:
+    """
+    Convert DFN files in `inpath` to TOML files in `outdir`.
+    By default, convert the definitions to schema version 2.
+    """
+    inpath = Path(inpath).expanduser().absolute()
+    outdir = Path(outdir).expanduser().absolute()
+    outdir.mkdir(exist_ok=True, parents=True)
+
+    if inpath.is_file():
+        if inpath.name == "common.dfn":
+            raise ValueError("Cannot convert common.dfn as a standalone file")
+
+        common_path = inpath.parent / "common.dfn"
+        if common_path.exists():
+            with common_path.open() as f:
+                common, _ = parse_dfn(f)
+        else:
+            common = {}
+
+        with inpath.open() as f:
+            dfn = load(f, name=inpath.stem, common=common, format="dfn")
+
+        dfn = map(dfn, schema_version=schema_version)
+        _convert(dfn, outdir / f"{inpath.stem}.toml")
+    else:
+        dfns = {
+            name: map(dfn, schema_version=schema_version) for name, dfn in load_flat(inpath).items()
+        }
+        tree = to_tree(dfns)
+        flat = to_flat(tree)
+        for dfn_name, dfn in flat.items():
+            _convert(dfn, outdir / f"{dfn_name}.toml")
+
+
+def _convert(dfn: Dfn, outpath: Path) -> None:
+    with Path.open(outpath, "wb") as f:
+        # TODO if we start using c/attrs, swap out
+        # all this for a custom unstructuring hook
+        dfn_dict = asdict(dfn)
+        dfn_dict["schema_version"] = str(dfn_dict["schema_version"])
+        if blocks := dfn_dict.pop("blocks", None):
+            for block_name, block_fields in blocks.items():
+                if block_name not in dfn_dict:
+                    dfn_dict[block_name] = {}
+                for field_name, field_data in block_fields.items():
+                    dfn_dict[block_name][field_name] = field_data
+
+        tomli.dump(
+            dict(
+                sorted(
+                    remap(dfn_dict, visit=drop_none_or_empty).items(),
+                    key=block_sort_key,
+                )
+            ),
+            f,
+        )
+
+
+if __name__ == "__main__":
+    """
+    Convert DFN files in the original format and schema version 1
+    to TOML files, by default also converting to schema version 2.
+    """
+
+    parser = argparse.ArgumentParser(
+        description="Convert DFN files to TOML.",
+        epilog=textwrap.dedent(
+            """\
+Convert DFN files in the original format and schema version 1
+to TOML files, by default also converting to schema version 2.
+"""
+        ),
+    )
+    parser.add_argument(
+        "--indir",
+        "-i",
+        type=str,
+        help="Directory containing DFN files, or a single DFN file.",
+    )
+    parser.add_argument(
+        "--outdir",
+        "-o",
+        help="Output directory.",
+    )
+    parser.add_argument(
+        "--schema-version",
+        "-s",
+        type=str,
+        default="2",
+        help="Schema version to convert to.",
+    )
+    parser.add_argument(
+        "--validate",
+        "-v",
+        action="store_true",
+        help="Validate DFN files without converting them.",
+    )
+    args = parser.parse_args()
+
+    if args.validate:
+        if not is_valid(args.indir):
+            sys.exit(1)
+    else:
+        convert(args.indir, args.outdir, args.schema_version)