fix(models): include files in nested folders (#201)

Some of the model directories have nested folders containing direct comparison files, input files for comparison models, or other supporting data.

Author: wpbonelli

.gitignore

@@ -149,3 +149,6 @@ autotest/temp/
 
 # uv lockfile
 uv.lock
+
+# filelock files
+**/*.lock

autotest/test_models.py

@@ -44,11 +44,13 @@ def test_get_examples(example_name, model_names):
 
 @pytest.mark.parametrize("model_name, files", list(islice(MODELMAP.items(), TAKE)))
 def test_copy_to(model_name, files, tmp_path):
-    workspace = models.copy_to(tmp_path, model_name)
+    workspace = models.copy_to(tmp_path, model_name, verbose=True)
     assert workspace.exists(), f"Model {model_name} was not copied to {tmp_path}"
     assert workspace.is_dir(), f"Model {model_name} is not a directory"
-    assert len(list(workspace.iterdir())) == len(files), (
-        f"Model {model_name} does not have the correct number of files"
+    found = [p for p in workspace.rglob("*") if p.is_file()]
+    assert len(found) == len(files), (
+        f"Model {model_name} does not have the correct number of files, "
+        f"expected {len(files)}, got {len(found)}"
     )
     if "mf6" in model_name:
         assert any(Path(f).name == "mfsim.nam" for f in files)

modflow_devtools/make_registry.py

@@ -45,36 +45,30 @@ def write_registry(
     if is_zip := url.endswith((".zip", ".tar")):
         registry[url.rpartition("/")[2]] = {"hash": None, "url": url}
 
-    def _find_examples_dir(p):
-        while p.name != "examples":
-            p = p.parent
-        return p
-
     model_paths = get_model_paths(path)
     for model_path in model_paths:
         # TODO: the renaming is only necessary because we're attaching auto-
         # generated functions to the models module. If we can live without,
         # and get by with a single function that takes model name as an arg,
         # then the model names could correspond directly to directory names.
         model_path = model_path.expanduser().absolute()
-        base_path = _find_examples_dir(model_path) if is_zip else path
-        rel_path = model_path.relative_to(base_path)
+        rel_path = model_path.relative_to(path)
         model_name = "/".join(rel_path.parts)
         modelmap[model_name] = []
         if is_zip:
             if rel_path.parts[0] not in examples:
                 examples[rel_path.parts[0]] = []
             examples[rel_path.parts[0]].append(model_name)
-        for p in model_path.glob("*"):
+        for p in model_path.rglob("*"):
             if not p.is_file() or any(e in p.name for e in exclude):
                 continue
             if is_zip:
-                relpath = p.expanduser().absolute().relative_to(base_path)
+                relpath = p.expanduser().absolute().relative_to(path)
                 name = "/".join(relpath.parts)
                 url_ = url
                 hash = None
             else:
-                relpath = p.expanduser().absolute().relative_to(base_path)
+                relpath = p.expanduser().absolute().relative_to(path)
                 name = "/".join(relpath.parts)
                 url_ = f"{url}/{relpath!s}"
                 hash = _sha256(p)

modflow_devtools/models.py

@@ -120,12 +120,19 @@ def copy_to(
 ) -> Path | None:
     if not any(files := FETCHERS[model_name]()):
         return None
+    # create the workspace if needed
     workspace = Path(workspace).expanduser().absolute()
     if verbose:
         print(f"Creating workspace {workspace}")
     workspace.mkdir(parents=True, exist_ok=True)
+    # copy the files. some might be in nested folders,
+    # but the first is guaranteed not to be, so use it
+    # to determine relative path in the new workspace.
+    base = Path(files[0]).parent
     for file in files:
         if verbose:
             print(f"Copying {file} to workspace")
-        copy(file, workspace)
+        path = workspace / file.relative_to(base)
+        path.parent.mkdir(parents=True, exist_ok=True)
+        copy(file, workspace / file.relative_to(base))
     return workspace

modflow_devtools/registry/examples.toml

@@ -56,19 +56,19 @@ ex-gwf-csub-p02b = [
 ]
 ex-gwf-csub-p02c = [
     "ex-gwf-csub-p02c/es-001",
-    "ex-gwf-csub-p02c/es-020",
+    "ex-gwf-csub-p02c/hb-001",
     "ex-gwf-csub-p02c/hb-002",
-    "ex-gwf-csub-p02c/hb-100",
-    "ex-gwf-csub-p02c/hb-050",
+    "ex-gwf-csub-p02c/es-100",
     "ex-gwf-csub-p02c/hb-005",
+    "ex-gwf-csub-p02c/hb-100",
+    "ex-gwf-csub-p02c/es-020",
+    "ex-gwf-csub-p02c/es-010",
+    "ex-gwf-csub-p02c/hb-020",
     "ex-gwf-csub-p02c/hb-010",
     "ex-gwf-csub-p02c/es-050",
-    "ex-gwf-csub-p02c/hb-001",
     "ex-gwf-csub-p02c/es-005",
+    "ex-gwf-csub-p02c/hb-050",
     "ex-gwf-csub-p02c/es-002",
-    "ex-gwf-csub-p02c/es-100",
-    "ex-gwf-csub-p02c/hb-020",
-    "ex-gwf-csub-p02c/es-010",
 ]
 ex-gwf-csub-p03a = [
     "ex-gwf-csub-p03a",