From 5d8d84596fc861fe30203ed402ef37dc8a27c139 Mon Sep 17 00:00:00 2001 From: Bonelli Date: Tue, 10 Feb 2026 14:21:07 -0500 Subject: [PATCH 1/4] consolidate --tag option into --version --- docs/md/dev/programs.md | 8 +++--- modflow_devtools/programs/make_registry.py | 33 ++++++++-------------- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/docs/md/dev/programs.md b/docs/md/dev/programs.md index 02f1cdae..433efb61 100644 --- a/docs/md/dev/programs.md +++ b/docs/md/dev/programs.md @@ -394,8 +394,8 @@ Exposed as a CLI command and Python API: # Sync all configured sources and release tags python -m modflow_devtools.programs sync -# Sync specific source to specific release tag -python -m modflow_devtools.programs sync --repo MODFLOW-ORG/modflow6 --tag 6.6.3 +# Sync specific source to specific release version +python -m modflow_devtools.programs sync --repo MODFLOW-ORG/modflow6 --version 6.6.3 # Force re-download python -m modflow_devtools.programs sync --force @@ -416,7 +416,7 @@ from modflow_devtools.programs import sync_registries, get_sync_status sync_registries() # Sync specific -sync_registries(repo="MODFLOW-ORG/modflow6", tag="6.6.3") +sync_registries(repo="MODFLOW-ORG/modflow6", version="6.6.3") # Check status status = get_sync_status() @@ -881,7 +881,7 @@ The Programs API has been implemented following a consolidated object-oriented a ```bash python -m modflow_devtools.programs.make_registry \ --repo MODFLOW-ORG/modflow6 \ - --tag 6.6.3 \ + --version 6.6.3 \ --programs mf6 zbud6 libmf6 mf5to6 \ --compute-hashes \ --output programs.toml diff --git a/modflow_devtools/programs/make_registry.py b/modflow_devtools/programs/make_registry.py index 3904b6dd..49bd1291 100644 --- a/modflow_devtools/programs/make_registry.py +++ b/modflow_devtools/programs/make_registry.py @@ -31,7 +31,7 @@ def compute_sha256(file_path: Path) -> str: return sha256.hexdigest() -def get_release_assets(repo: str, tag: str) -> list[dict]: +def get_release_assets(repo: str, version: str) -> list[dict]: """ Get release assets for a GitHub release. @@ -39,15 +39,15 @@ def get_release_assets(repo: str, tag: str) -> list[dict]: ---------- repo : str Repository in "owner/name" format - tag : str - Release tag + version : str + Release version (tag) Returns ------- list[dict] List of asset dictionaries from GitHub API """ - url = f"https://api.github.com/repos/{repo}/releases/tags/{tag}" + url = f"https://api.github.com/repos/{repo}/releases/tags/{version}" response = requests.get(url) response.raise_for_status() release_data = response.json() @@ -82,7 +82,7 @@ def main(): # Generate registry from existing GitHub release (for testing) python -m modflow_devtools.programs.make_registry \\ --repo MODFLOW-ORG/modflow6 \\ - --tag 6.6.3 \\ + --version 6.6.3 \\ --programs mf6 zbud6 libmf6 mf5to6 \\ --output programs.toml @@ -101,12 +101,6 @@ def main(): type=str, help='Repository in "owner/name" format (e.g., MODFLOW-ORG/modflow6) [required]', ) - parser.add_argument( - "--tag", - required=False, - type=str, - help="Release tag (e.g., 6.6.3) [required when scanning GitHub release]", - ) parser.add_argument( "--dists", type=str, @@ -121,7 +115,7 @@ def main(): parser.add_argument( "--version", required=False, - help="Program version [required when using --dists, defaults to --tag otherwise]", + help="Program version", ) parser.add_argument( "--description", @@ -162,8 +156,8 @@ def main(): sys.exit(1) else: # GitHub release mode - if not args.repo or not args.tag: - print("Error: --repo and --tag are required when not using --dists", file=sys.stderr) + if not args.repo: + print("Error: --repo is required when not using --dists", file=sys.stderr) sys.exit(1) # Parse programs (support name:path syntax) @@ -199,16 +193,16 @@ def main(): else: # GitHub release mode: fetch from GitHub API if args.verbose: - print(f"Fetching release assets for {args.repo}@{args.tag}...") + print(f"Fetching release assets for {args.repo}@{args.version}...") try: - assets = get_release_assets(args.repo, args.tag) + assets = get_release_assets(args.repo, args.version) except Exception as e: print(f"Error fetching release assets: {e}", file=sys.stderr) sys.exit(1) if not assets: - print(f"No assets found for release {args.tag}", file=sys.stderr) + print(f"No assets found for release {args.version}", file=sys.stderr) sys.exit(1) if args.verbose: @@ -222,9 +216,6 @@ def main(): "programs": {}, } - # Use tag as version if not specified - version = args.version or args.tag - # Distribution name mappings for filenames dist_map = { "linux": ["linux", "ubuntu"], @@ -246,7 +237,7 @@ def main(): print(f"\nProcessing program: {program_name}") program_meta = { - "version": version, + "version": args.version, "repo": args.repo, "exe": program_exes[program_name], # Get exe path for this program } From 2a9ec0f59d5b028e2dd3284f341b7cf6e80c2204 Mon Sep 17 00:00:00 2001 From: Bonelli Date: Tue, 10 Feb 2026 14:42:46 -0500 Subject: [PATCH 2/4] leaner registry file config --- docs/md/dev/models.md | 3 - docs/md/dev/programs.md | 64 +++++++------ modflow_devtools/models/__init__.py | 18 ---- modflow_devtools/programs/__init__.py | 102 ++++++++++----------- modflow_devtools/programs/make_registry.py | 16 ++-- 5 files changed, 91 insertions(+), 112 deletions(-) diff --git a/docs/md/dev/models.md b/docs/md/dev/models.md index 565c8cf9..86db5512 100644 --- a/docs/md/dev/models.md +++ b/docs/md/dev/models.md @@ -181,10 +181,7 @@ The registry file contains: Example `models.toml`: ```toml -# Metadata (top-level) schema_version = "1.0" -generated_at = "2025-12-04T14:30:00Z" -devtools_version = "1.9.0" [files] "ex-gwf-twri01/mfsim.nam" = {hash = "sha256:abc123..."} diff --git a/docs/md/dev/programs.md b/docs/md/dev/programs.md index 433efb61..e70fe844 100644 --- a/docs/md/dev/programs.md +++ b/docs/md/dev/programs.md @@ -168,45 +168,54 @@ Each source repository must make a **program registry** file available. Program Registry files shall be named **`programs.toml`** (not `registry.toml` - the specific naming distinguishes it from the Models and DFNs registries) and contain, at minimum, a dictionary `programs` enumerating programs provided by the source repository. For instance: ```toml -# Metadata (top-level) schema_version = "1.0" -generated_at = "2025-12-29T10:30:00Z" -devtools_version = "2.0.0" [programs.mf6] -version = "6.6.3" +# Optional: exe defaults to "bin/mf6" (or "bin/mf6.exe" on Windows), only specify if different description = "MODFLOW 6 groundwater flow model" -repo = "MODFLOW-ORG/modflow6" license = "CC0-1.0" -[programs.mf6.binaries.linux] +[[programs.mf6.dists]] +name = "linux" asset = "mf6.6.3_linux.zip" hash = "sha256:..." -exe = "bin/mf6" -[programs.mf6.binaries.mac] -asset = "mf6.6.3_macarm.zip" +[[programs.mf6.dists]] +name = "mac" +asset = "mf6.6.3_mac.zip" hash = "sha256:..." -exe = "bin/mf6" -[programs.mf6.binaries.win64] +[[programs.mf6.dists]] +name = "win64" asset = "mf6.6.3_win64.zip" hash = "sha256:..." -exe = "bin/mf6.exe" [programs.zbud6] -version = "6.6.3" +# exe defaults to "bin/zbud6" (or "bin/zbud6.exe" on Windows) description = "MODFLOW 6 Zonebudget utility" -repo = "MODFLOW-ORG/modflow6" license = "CC0-1.0" -[programs.zbud6.binaries.linux] +[[programs.zbud6.dists]] +name = "linux" asset = "mf6.6.3_linux.zip" hash = "sha256:..." -exe = "bin/zbud6" + +[[programs.zbud6.dists]] +name = "mac" +asset = "mf6.6.3_mac.zip" +hash = "sha256:..." + +[[programs.zbud6.dists]] +name = "win64" +asset = "mf6.6.3_win64.zip" +hash = "sha256:..." ``` -The top-level metadata is optional; if a `schema_version` is not provided it will be inferred if possible. +**Simplified format notes**: +- Version and repository information come from the release tag and bootstrap configuration, not from the registry file +- The `exe` field is optional and defaults to `bin/{program}` (with `.exe` automatically added on Windows) +- Only specify `exe` when the executable location differs from the default +- The `schema_version` field is optional but recommended for future compatibility Platform identifiers are as defined in the [modflow-devtools OS tag specification](https://modflow-devtools.readthedocs.io/en/latest/md/ostags.html): `linux`, `mac`, `win64`. @@ -573,31 +582,32 @@ Examples: The Programs API uses a consolidated object-oriented design with Pydantic models and concrete classes. -#### ProgramBinary +#### ProgramDistribution -Represents platform-specific binary information: +Represents platform-specific distribution information: ```python -class ProgramBinary(BaseModel): - """Platform-specific binary information.""" +class ProgramDistribution(BaseModel): + """Distribution-specific information.""" + name: str # Distribution name (e.g., linux, mac, win64) asset: str # Release asset filename hash: str | None # SHA256 hash - exe: str # Executable path within archive ``` #### ProgramMetadata Program metadata in registry: -Example: ```python class ProgramMetadata(BaseModel): """Program metadata in registry.""" - version: str description: str | None - repo: str # Source repository (owner/name) license: str | None - binaries: dict[str, ProgramBinary] # Platform-specific binaries + exe: str | None # Optional: defaults to bin/{program} + dists: list[ProgramDistribution] # Available distributions + + def get_exe_path(self, program_name: str, platform: str | None = None) -> str: + """Get executable path, using default if not specified.""" ``` #### ProgramRegistry @@ -608,8 +618,6 @@ Top-level registry data model: class ProgramRegistry(BaseModel): """Program registry data model.""" schema_version: str | None - generated_at: datetime | None - devtools_version: str | None programs: dict[str, ProgramMetadata] ``` diff --git a/modflow_devtools/models/__init__.py b/modflow_devtools/models/__init__.py index 347b53cd..a6436f0d 100644 --- a/modflow_devtools/models/__init__.py +++ b/modflow_devtools/models/__init__.py @@ -3,7 +3,6 @@ import urllib from collections.abc import Callable from dataclasses import dataclass, field -from datetime import datetime, timezone from functools import partial from os import PathLike from pathlib import Path @@ -150,10 +149,6 @@ class ModelRegistry(BaseModel): """ schema_version: str | None = Field(None, description="Registry schema version") - generated_at: datetime | None = Field(None, description="Timestamp when registry was generated") - devtools_version: str | None = Field( - None, description="Version of modflow-devtools used to generate" - ) files: dict[str, ModelInputFile] = Field( default_factory=dict, description="Map of file names to file entries" ) @@ -166,11 +161,6 @@ class ModelRegistry(BaseModel): model_config = {"arbitrary_types_allowed": True, "populate_by_name": True} - @field_serializer("generated_at") - def serialize_datetime(self, dt: datetime | None, _info): - """Serialize datetime to ISO format string.""" - return dt.isoformat() if dt is not None else None - def copy_to( self, workspace: str | PathLike, model_name: str, verbose: bool = False ) -> Path | None: @@ -871,8 +861,6 @@ def __init__(self) -> None: # Initialize Pydantic parent with empty data (no metadata for local registries) super().__init__( schema_version=None, - generated_at=None, - devtools_version=None, files={}, models={}, examples={}, @@ -1015,8 +1003,6 @@ def __init__( # Initialize Pydantic parent with empty data (will be populated by _load()) super().__init__( schema_version=None, - generated_at=None, - devtools_version=None, files={}, models={}, examples={}, @@ -1114,8 +1100,6 @@ def _try_load_from_cache(self) -> bool: # Store metadata from first registry if not self.schema_version and registry.schema_version: self.schema_version = registry.schema_version - self.generated_at = registry.generated_at - self.devtools_version = registry.devtools_version if not self.files: return False @@ -1244,8 +1228,6 @@ def index( registry_data = { "schema_version": "1.0", - "generated_at": datetime.now(timezone.utc).isoformat(), - "devtools_version": modflow_devtools.__version__, "files": remap(dict(sorted(existing_files.items())), visit=drop_none_or_empty), "models": dict(sorted(existing_models.items())), "examples": dict(sorted(existing_examples.items())), diff --git a/modflow_devtools/programs/__init__.py b/modflow_devtools/programs/__init__.py index 8bdf53a9..f6cefddf 100644 --- a/modflow_devtools/programs/__init__.py +++ b/modflow_devtools/programs/__init__.py @@ -1,25 +1,3 @@ -""" -Programs API - Dynamic program registry and installation management. - -This module provides utilities for discovering, synchronizing, and managing -MODFLOW-related programs. It follows the same design patterns as the Models API -with a consolidated object-oriented implementation. - -Key classes: - - ProgramCache: Manages local caching of registries - - ProgramSourceRepo: Represents a program source repository - - ProgramSourceConfig: Configuration container from bootstrap file - - ProgramRegistry: Pydantic model for registry structure - - DiscoveredProgramRegistry: Discovery result with metadata - -Example usage: - >>> from modflow_devtools.programs import ProgramSourceConfig - >>> config = ProgramSourceConfig.load() - >>> source = config.sources["modflow6"] - >>> result = source.sync(ref="6.6.3", verbose=True) - >>> # Use _DEFAULT_CACHE to access cached registries -""" - import hashlib import os import shutil @@ -33,7 +11,7 @@ import tomli import tomli_w from filelock import FileLock -from pydantic import BaseModel, Field, field_serializer +from pydantic import BaseModel, Field _CACHE_ROOT = Path(pooch.os_cache("modflow-devtools")) """Root cache directory (platform-appropriate location via Pooch)""" @@ -90,35 +68,56 @@ class ProgramDistribution(BaseModel): class ProgramMetadata(BaseModel): """Program metadata in registry.""" - version: str = Field(..., description="Program version") description: str | None = Field(None, description="Program description") - repo: str = Field(..., description="Source repository (owner/name)") license: str | None = Field(None, description="License identifier") - exe: str = Field(..., description="Executable path within archive (e.g., bin/mf6)") + exe: str | None = Field( + None, + description="Executable path within archive (e.g., bin/mf6). Defaults to bin/{program}", + ) dists: list[ProgramDistribution] = Field( default_factory=list, description="Available distributions" ) model_config = {"arbitrary_types_allowed": True} + def get_exe_path(self, program_name: str, platform: str | None = None) -> str: + """ + Get executable path, using default if not specified. + + Parameters + ---------- + program_name : str + Name of the program + platform : str | None + Platform name (e.g., 'win64'). If Windows platform, adds .exe extension. + + Returns + ------- + str + Executable path within archive + """ + if self.exe: + exe = self.exe + else: + exe = f"bin/{program_name}" + + # Add .exe extension for Windows platforms + if platform and platform.startswith("win") and not exe.endswith(".exe"): + exe = f"{exe}.exe" + + return exe + class ProgramRegistry(BaseModel): """Program registry data model.""" schema_version: str | None = Field(None, description="Registry schema version") - generated_at: datetime | None = Field(None, description="Generation timestamp") - devtools_version: str | None = Field(None, description="modflow-devtools version") programs: dict[str, ProgramMetadata] = Field( default_factory=dict, description="Map of program names to metadata" ) model_config = {"arbitrary_types_allowed": True, "populate_by_name": True} - @field_serializer("generated_at") - def serialize_datetime(self, dt: datetime | None, _info): - """Serialize datetime to ISO format.""" - return dt.isoformat() if dt is not None else None - @dataclass class DiscoveredProgramRegistry: @@ -1187,16 +1186,17 @@ def install( # Search all cached registries for the program found_registry: ProgramRegistry | None = None found_ref: str | None = None + found_source: ProgramSourceRepo | None = None for source_name, source in config.sources.items(): for ref in source.refs: registry = self.cache.load(source_name, ref) if registry and program in registry.programs: - # If version specified, check if it matches - prog_meta = registry.programs[program] - if version is None or prog_meta.version == version: + # If version specified, check if it matches the ref (release tag) + if version is None or ref == version: found_registry = registry found_ref = ref + found_source = source break if found_registry: break @@ -1213,10 +1213,11 @@ def install( for ref in source.refs: registry = self.cache.load(source_name, ref) if registry and program in registry.programs: - prog_meta = registry.programs[program] - if version is None or prog_meta.version == version: + # If version specified, check if it matches the ref (release tag) + if version is None or ref == version: found_registry = registry found_ref = ref + found_source = source break if found_registry: break @@ -1231,7 +1232,8 @@ def install( # 2. Get program metadata program_meta = found_registry.programs[program] - version = program_meta.version # Use actual version from registry + assert found_source is not None # Guaranteed by found_registry check above + version = found_ref # Use release tag as version if verbose: print(f"Installing {program} version {version}...") @@ -1277,14 +1279,12 @@ def install( if verbose: print(f"{program} {version} is already installed in {bindir}") # Return paths to existing executables - exe_name = Path(program_meta.exe).name - # Add .exe extension on Windows - if platform.startswith("win") and not exe_name.endswith(".exe"): - exe_name += ".exe" + exe_path = program_meta.get_exe_path(program, platform) + exe_name = Path(exe_path).name return [bindir / exe_name] # 7. Download archive (if not cached) - asset_url = f"https://github.com/{program_meta.repo}/releases/download/{found_ref}/{dist_meta.asset}" + asset_url = f"https://github.com/{found_source.repo}/releases/download/{found_ref}/{dist_meta.asset}" archive_dir = self.cache.get_archive_dir(program, version, platform) archive_path = archive_dir / dist_meta.asset @@ -1301,10 +1301,7 @@ def install( # 8. Extract to binaries cache (if not already extracted) binary_dir = self.cache.get_binary_dir(program, version, platform) - exe_path = program_meta.exe - # Add .exe extension on Windows for extraction path - if platform.startswith("win") and not exe_path.endswith(".exe"): - exe_path += ".exe" + exe_path = program_meta.get_exe_path(program, platform) exe_in_cache = binary_dir / exe_path if not exe_in_cache.exists() or force: @@ -1319,10 +1316,9 @@ def install( ) # 9. Copy executables to bindir - exe_name = Path(program_meta.exe).name - # Add .exe extension on Windows - if platform.startswith("win") and not exe_name.endswith(".exe"): - exe_name += ".exe" + exe_name = Path( + exe_path + ).name # exe_path already set above with platform-specific extension dest_exe = bindir / exe_name if verbose: @@ -1340,7 +1336,7 @@ def install( # 10. Update metadata assert found_ref is not None # Guaranteed by found_registry check above source_info: dict[str, str] = { - "repo": program_meta.repo, + "repo": found_source.repo, "tag": found_ref, "asset_url": asset_url, "hash": dist_meta.hash or "", diff --git a/modflow_devtools/programs/make_registry.py b/modflow_devtools/programs/make_registry.py index 49bd1291..3c391c22 100644 --- a/modflow_devtools/programs/make_registry.py +++ b/modflow_devtools/programs/make_registry.py @@ -12,15 +12,12 @@ import hashlib import sys import tempfile -from datetime import datetime, timezone from glob import glob from pathlib import Path import requests # type: ignore[import-untyped] import tomli_w -import modflow_devtools - def compute_sha256(file_path: Path) -> str: """Compute SHA256 hash of a file.""" @@ -211,8 +208,6 @@ def main(): # Build registry structure registry = { "schema_version": "1.0", - "generated_at": datetime.now(timezone.utc).isoformat(), - "devtools_version": modflow_devtools.__version__, "programs": {}, } @@ -236,11 +231,12 @@ def main(): if args.verbose: print(f"\nProcessing program: {program_name}") - program_meta = { - "version": args.version, - "repo": args.repo, - "exe": program_exes[program_name], # Get exe path for this program - } + program_meta = {} + + # Only include exe if it differs from the default (bin/{program_name}) + exe_path = program_exes[program_name] + if exe_path != f"bin/{program_name}": + program_meta["exe"] = exe_path if args.description: program_meta["description"] = args.description From c7dd4936461b62fc4f6039f0728ff2d241d7fbaf Mon Sep 17 00:00:00 2001 From: Bonelli Date: Tue, 10 Feb 2026 14:57:41 -0500 Subject: [PATCH 3/4] appease mypy --- modflow_devtools/programs/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modflow_devtools/programs/__init__.py b/modflow_devtools/programs/__init__.py index f6cefddf..2916cafb 100644 --- a/modflow_devtools/programs/__init__.py +++ b/modflow_devtools/programs/__init__.py @@ -1233,6 +1233,7 @@ def install( # 2. Get program metadata program_meta = found_registry.programs[program] assert found_source is not None # Guaranteed by found_registry check above + assert found_ref is not None # Guaranteed by found_registry check above version = found_ref # Use release tag as version if verbose: From f42a0bb096edcff3862bd9d4392ad64e72466ae9 Mon Sep 17 00:00:00 2001 From: Bonelli Date: Tue, 10 Feb 2026 15:38:36 -0500 Subject: [PATCH 4/4] fix test --- autotest/test_models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/autotest/test_models.py b/autotest/test_models.py index fb00a282..a20b03c6 100644 --- a/autotest/test_models.py +++ b/autotest/test_models.py @@ -414,11 +414,7 @@ def synced_registry(self): def test_registry_has_metadata(self, synced_registry): """Test that registry has required metadata.""" assert hasattr(synced_registry, "schema_version") - assert hasattr(synced_registry, "generated_at") - assert hasattr(synced_registry, "devtools_version") assert synced_registry.schema_version is not None - assert synced_registry.generated_at is not None - assert synced_registry.devtools_version is not None def test_registry_has_files(self, synced_registry): """Test that registry has files."""