diff --git a/autotest/test_programs.py b/autotest/test_programs.py index 761c47f1..ce5daf61 100644 --- a/autotest/test_programs.py +++ b/autotest/test_programs.py @@ -256,18 +256,14 @@ def test_default_manager_exists(self): def test_convenience_wrappers(self): """Test that convenience functions wrap the default manager.""" from modflow_devtools.programs import ( - get_executable, install_program, list_installed, - select_version, uninstall_program, ) # All functions should exist and be callable assert callable(install_program) - assert callable(select_version) assert callable(uninstall_program) - assert callable(get_executable) assert callable(list_installed) def test_program_manager_list_installed_empty(self): @@ -301,10 +297,6 @@ def test_program_manager_error_handling(self): with pytest.raises(ProgramInstallationError, match="not found"): manager.install("nonexistent-program-xyz") - # Test get_executable for non-installed program - with pytest.raises(ProgramInstallationError, match="not installed"): - manager.get_executable("nonexistent-program-xyz") - def test_installation_metadata_integration(self): """Test InstallationMetadata integration with ProgramManager.""" from datetime import datetime, timezone @@ -333,7 +325,6 @@ def test_installation_metadata_integration(self): "hash": "", }, executables=["test-program"], - active=True, ) metadata.add_installation(installation) @@ -344,7 +335,6 @@ def test_installation_metadata_integration(self): assert len(installations) == 1 assert installations[0].version == "1.0.0" assert installations[0].platform == "linux" - assert installations[0].active is True # Clean up cache.clear() diff --git a/docs/md/dev/programs.md b/docs/md/dev/programs.md index 4902dc09..c0b790b6 100644 --- a/docs/md/dev/programs.md +++ b/docs/md/dev/programs.md @@ -232,17 +232,61 @@ hash = "sha256:..." The `exe` field can be specified at three levels, checked in this order: 1. **Distribution-level** (`[[programs.{name}.dists]]` entry with `exe` field) + - **Supports any custom path** within the archive - Use when different platforms have different archive structures - Most specific - overrides program-level and default - Example: `exe = "mf6.7.0_win64/bin/mf6.exe"` + - Example: `exe = "custom/nested/path/to/program"` 2. **Program-level** (`[programs.{name}]` section with `exe` field) + - **Supports any custom path** shared across all platforms - Use when all platforms share the same relative path structure - Example: `exe = "bin/mfnwt"` + - Example: `exe = "special/location/program"` 3. **Default** (neither specified) - - Falls back to `bin/{program}` - - Example: For `mf6`, defaults to `bin/mf6` + - **Automatically detects** executable location when installing + - Tries common patterns in order: + - **Nested with bin/**: `{archive_name}/bin/{program}` + - **Nested without bin/**: `{archive_name}/{program}` + - **Flat with bin/**: `bin/{program}` + - **Flat without bin/**: `{program}` + - Example: For `mf6`, automatically finds binary whether in `mf6.7.0_linux/bin/mf6`, `bin/mf6`, or other common layouts + - Only used when no explicit `exe` field is provided + +**Archive structure patterns**: + +The API supports four common archive layouts: + +1. **Nested with bin/** (e.g., MODFLOW 6): + ``` + mf6.7.0_linux.zip + └── mf6.7.0_linux/ + └── bin/ + └── mf6 + ``` + +2. **Nested without bin/**: + ``` + program.1.0_linux.zip + └── program.1.0_linux/ + └── program + ``` + +3. **Flat with bin/**: + ``` + program.zip + └── bin/ + └── program + ``` + +4. **Flat without bin/**: + ``` + program.zip + └── program + ``` + +The `make_registry` tool automatically detects which pattern each archive uses and only stores non-default exe paths in the registry. **Windows .exe extension handling**: - The `.exe` extension is automatically added on Windows platforms if not present @@ -628,6 +672,12 @@ python -m modflow_devtools.programs.make_registry \ - Optionally computes SHA256 hashes from local files with `--compute-hashes` - Creates asset entries from local file names - Auto-detects platform from file names (linux, mac, win64, etc.) +- **Automatic pattern detection**: + - Inspects archives to detect executable locations + - Recognizes nested and flat archive patterns + - Automatically optimizes exe paths (only stores non-default paths) + - Detects when all distributions use the same relative path + - Caches downloaded assets to avoid redundant downloads when multiple programs share the same archive **Example CI integration** (GitHub Actions): ```yaml @@ -662,9 +712,15 @@ python -m modflow_devtools.programs.make_registry \ **How it works:** - Fetches release assets from GitHub API using repo and version (tag) -- Downloads assets if `--compute-hashes` is specified +- Downloads assets to detect exe paths and enable pattern optimization +- Optionally computes SHA256 hashes with `--compute-hashes` - Useful for testing or regenerating a registry for an existing release - No `--dists` argument needed - pulls from GitHub directly +- **Automatic pattern detection** (same as Mode 1): + - Inspects archives to find executables + - Detects nested/flat patterns automatically + - Only stores non-default exe paths in registry + - Caches downloads when processing multiple programs from same release **Additional options:** ```bash diff --git a/modflow_devtools/programs/__init__.py b/modflow_devtools/programs/__init__.py index ddfc4b85..d7c5a6ac 100644 --- a/modflow_devtools/programs/__init__.py +++ b/modflow_devtools/programs/__init__.py @@ -83,7 +83,13 @@ class ProgramMetadata(BaseModel): model_config = {"arbitrary_types_allowed": True} - def get_exe_path(self, program_name: str, platform: str | None = None) -> str: + def get_exe_path( + self, + program_name: str, + platform: str | None = None, + asset_name: str | None = None, + archive_path: Path | None = None, + ) -> str: """ Get executable path, using default if not specified. @@ -93,34 +99,127 @@ def get_exe_path(self, program_name: str, platform: str | None = None) -> str: Name of the program platform : str | None Platform name (e.g., 'win64'). If Windows platform, adds .exe extension. + asset_name : str | None + Asset filename (e.g., 'mf6.6.3_linux.zip'). If provided and program-level exe is used, + prepends the asset stem (filename without extension) to support nested folder structure. + archive_path : Path | None + Path to archive file. If provided and using defaults, will inspect archive to determine + which default pattern to use (bin/{program} vs {program} at root). Returns ------- str Executable path within archive """ + exe: str | None + # Check distribution-specific exe path first if platform: for dist in self.dists: if dist.name == platform and dist.exe: exe = dist.exe # Add .exe extension for Windows platforms if not present - if platform.startswith("win") and not exe.endswith(".exe"): + if platform.startswith("win") and exe and not exe.endswith(".exe"): exe = f"{exe}.exe" + assert exe is not None # Narrowing for mypy return exe - # Fall back to program-level exe or default + # Fall back to program-level exe or defaults if self.exe: exe = self.exe else: + # Try common defaults in order: + # 1. bin/{program} (most common) + # 2. {program} (binaries at root) + + # If we have the archive, inspect it to determine which pattern to use + if archive_path and asset_name: + from pathlib import Path + + asset_stem = Path(asset_name).stem + + # Try to detect which default exists in the archive + exe = self._detect_default_exe_in_archive( + archive_path, asset_stem, program_name, platform + ) + if exe: + # Already includes asset stem prefix + return exe + + # Fallback when archive not available - try bin/ first exe = f"bin/{program_name}" # Add .exe extension for Windows platforms if platform and platform.startswith("win") and not exe.endswith(".exe"): exe = f"{exe}.exe" + # If asset_name provided and we're using program-level exe, + # prepend asset stem to support nested folder structure + # (e.g., 'bin/mf6' becomes 'mf6.6.3_linux/bin/mf6' for mf6.6.3_linux.zip) + if asset_name and not any( + dist.name == platform and dist.exe for dist in self.dists if platform + ): + # Using program-level exe (not dist-specific) + from pathlib import Path + + asset_stem = Path(asset_name).stem + exe = f"{asset_stem}/{exe}" + return exe + def _detect_default_exe_in_archive( + self, + archive_path: Path, + asset_stem: str, + program_name: str, + platform: str | None, + ) -> str | None: + """ + Inspect archive to detect which default exe pattern is used. + + Supports both nested ({asset_stem}/path) and flat (path) patterns. + Returns the full path if found, None otherwise. + """ + import tarfile + import zipfile + + # Try to list archive contents + try: + if archive_path.suffix.lower() == ".zip": + with zipfile.ZipFile(archive_path, "r") as zf: + members = zf.namelist() + elif archive_path.suffix.lower() in [".gz", ".tgz"]: + with tarfile.open(archive_path, "r:gz") as tf: + members = tf.getnames() + elif archive_path.suffix.lower() == ".tar": + with tarfile.open(archive_path, "r") as tf: + members = tf.getnames() + else: + return None + + # Normalize member paths + members_normalized = [m.replace("\\", "/") for m in members] + + # Try common patterns in priority order + # First try nested patterns (most common), then flat patterns + for base_pattern in [f"bin/{program_name}", program_name]: + for ext in ["", ".exe", ".dll", ".so", ".dylib"]: + search_pattern = f"{base_pattern}{ext}" + + # Try nested pattern first (asset_stem/path) + nested_pattern = f"{asset_stem}/{search_pattern}" + if nested_pattern in members_normalized: + return nested_pattern + + # Try flat pattern (no asset_stem prefix) + if search_pattern in members_normalized: + return search_pattern + + return None + + except (zipfile.BadZipFile, tarfile.TarError, OSError): + return None + class ProgramRegistry(BaseModel): """Program registry data model.""" @@ -934,9 +1033,6 @@ class ProgramInstallation: executables: list[str] """List of installed executable names""" - active: bool - """Whether this is the active version in this bindir""" - class InstallationMetadata: """Manages installation metadata for a program.""" @@ -987,7 +1083,6 @@ def load(self) -> bool: installed_at=installed_at, source=inst_data["source"], executables=inst_data["executables"], - active=inst_data.get("active", False), ) self.installations.append(installation) @@ -1014,7 +1109,6 @@ def save(self) -> None: "installed_at": inst.installed_at.isoformat(), "source": inst.source, "executables": inst.executables, - "active": inst.active, } for inst in self.installations ], @@ -1031,11 +1125,6 @@ def add_installation(self, installation: ProgramInstallation) -> None: ---------- installation : ProgramInstallation Installation to add - - Notes - ----- - If installation marks this version as active in a bindir, all other - versions in that bindir are marked inactive. """ # Remove existing installation for same version/bindir if present self.installations = [ @@ -1044,12 +1133,6 @@ def add_installation(self, installation: ProgramInstallation) -> None: if not (inst.version == installation.version and inst.bindir == installation.bindir) ] - # If marking as active, deactivate others in same bindir - if installation.active: - for inst in self.installations: - if inst.bindir == installation.bindir: - inst.active = False - self.installations.append(installation) self.save() @@ -1071,55 +1154,6 @@ def remove_installation(self, version: str, bindir: Path) -> None: ] self.save() - def get_active_version(self, bindir: Path) -> str | None: - """ - Get active version in a bindir. - - Parameters - ---------- - bindir : Path - Installation directory - - Returns - ------- - str | None - Active version, or None if no active installation - """ - for inst in self.installations: - if inst.bindir == bindir and inst.active: - return inst.version - return None - - def set_active_version(self, version: str, bindir: Path) -> None: - """ - Set active version in a bindir. - - Parameters - ---------- - version : str - Program version - bindir : Path - Installation directory - - Raises - ------ - ValueError - If version is not installed in bindir - """ - # Find the installation - found = False - for inst in self.installations: - if inst.version == version and inst.bindir == bindir: - inst.active = True - found = True - elif inst.bindir == bindir: - inst.active = False - - if not found: - raise ValueError(f"Version {version} is not installed in {bindir}") - - self.save() - def list_installations(self) -> list[ProgramInstallation]: """ List all installations. @@ -1288,16 +1322,6 @@ def install( metadata = InstallationMetadata(program) metadata.load() - if not force: - active_version = metadata.get_active_version(bindir) - if active_version == version: - if verbose: - print(f"{program} {version} is already installed in {bindir}") - # Return paths to existing executables - exe_path = program_meta.get_exe_path(program, platform) - exe_name = Path(exe_path).name - return [bindir / exe_name] - # 7. Download archive (if not cached) asset_url = f"https://github.com/{found_source.repo}/releases/download/{found_ref}/{dist_meta.asset}" archive_dir = self.cache.get_archive_dir(program, version, platform) @@ -1314,9 +1338,11 @@ def install( verbose=verbose, ) + # Get exe path (may inspect archive to detect defaults) + exe_path = program_meta.get_exe_path(program, platform, dist_meta.asset, archive_path) + # 8. Extract to binaries cache (if not already extracted) binary_dir = self.cache.get_binary_dir(program, version, platform) - exe_path = program_meta.get_exe_path(program, platform) exe_in_cache = binary_dir / exe_path if not exe_in_cache.exists() or force: @@ -1363,7 +1389,6 @@ def install( installed_at=datetime.now(timezone.utc), source=source_info, executables=[exe_name], - active=True, ) metadata.add_installation(installation) @@ -1374,112 +1399,6 @@ def install( # 11. Return installed executable paths return [dest_exe] - def select( - self, - program: str, - version: str, - bindir: str | Path | None = None, - verbose: bool = False, - ) -> list[Path]: - """ - Switch active version in bindir (re-copy from cache). - - Parameters - ---------- - program : str - Program name - version : str - Program version - bindir : str | Path, optional - Installation directory (default: use previous installation location) - verbose : bool - Print progress messages - - Returns - ------- - list[Path] - List of executable paths - - Raises - ------ - ProgramInstallationError - If version is not cached or bindir cannot be determined - """ - import shutil - - # Load metadata - metadata = InstallationMetadata(program) - if not metadata.load(): - raise ProgramInstallationError(f"No installation metadata found for {program}") - - # Determine bindir - if bindir is None: - bindir_options = get_bindir_options(program) - if not bindir_options: - raise ProgramInstallationError( - "No installation directory found. Specify bindir explicitly." - ) - bindir = bindir_options[0] - else: - bindir = Path(bindir) - - # Find installation with this version - installation = None - for inst in metadata.list_installations(): - if inst.version == version: - installation = inst - break - - if not installation: - raise ProgramInstallationError( - f"{program} version {version} is not installed. Run install() first." - ) - - # Check that binaries are cached - binary_dir = self.cache.get_binary_dir(program, version, installation.platform) - if not binary_dir.exists(): - raise ProgramInstallationError( - f"Binaries for {program} {version} not found in cache. " - f"Run install() to download and cache." - ) - - # Copy executables to bindir - dest_paths = [] - for exe_name in installation.executables: - src_exe = binary_dir / "bin" / exe_name # Assuming bin/ subdir - if not src_exe.exists(): - # Try without bin/ prefix - src_exe = binary_dir / exe_name - - if not src_exe.exists(): - raise ProgramInstallationError( - f"Executable {exe_name} not found in cache at {binary_dir}" - ) - - dest_exe = bindir / exe_name - - if verbose: - print(f"Copying {src_exe} to {dest_exe}...") - - shutil.copy2(src_exe, dest_exe) - - # Apply executable permissions on Unix - if os.name != "nt": - import stat - - current_permissions = dest_exe.stat().st_mode - dest_exe.chmod(current_permissions | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - dest_paths.append(dest_exe) - - # Update metadata (mark this version as active) - metadata.set_active_version(version, bindir) - - if verbose: - print(f"Activated {program} {version} in {bindir}") - - return dest_paths - def uninstall( self, program: str, @@ -1575,69 +1494,6 @@ def uninstall( if verbose: print(f"Successfully uninstalled {program}") - def get_executable( - self, - program: str, - version: str | None = None, - bindir: str | Path | None = None, - ) -> Path: - """ - Get path to installed executable. - - Parameters - ---------- - program : str - Program name - version : str, optional - Program version (default: active version in bindir) - bindir : str | Path, optional - Installation directory (default: search in priority order) - - Returns - ------- - Path - Path to executable - - Raises - ------ - ProgramInstallationError - If executable not found - """ - # Load metadata - metadata = InstallationMetadata(program) - if not metadata.load(): - raise ProgramInstallationError(f"Program {program} is not installed") - - # Determine bindir - if bindir is None: - bindir_options = get_bindir_options(program) - if bindir_options: - bindir = bindir_options[0] - else: - bindir = Path(bindir) - - # Find installation - for inst in metadata.list_installations(): - if bindir and inst.bindir != bindir: - continue - if version and inst.version != version: - continue - if bindir and not inst.active: - continue - - # Return first executable - if inst.executables: - exe_path = inst.bindir / inst.executables[0] - if exe_path.exists(): - return exe_path - - raise ProgramInstallationError( - f"Executable for {program}" - + (f" version {version}" if version else "") - + (f" in {bindir}" if bindir else "") - + " not found" - ) - def list_installed(self, program: str | None = None) -> dict[str, list[ProgramInstallation]]: """ List installed programs. @@ -1731,46 +1587,6 @@ def install_program( ) -def select_version( - program: str, - version: str, - bindir: str | Path | None = None, - verbose: bool = False, -) -> list[Path]: - """ - Switch active version in bindir (re-copy from cache). - - Convenience wrapper for ProgramManager.select(). - - Parameters - ---------- - program : str - Program name - version : str - Program version - bindir : str | Path, optional - Installation directory (default: use previous installation location) - verbose : bool - Print progress messages - - Returns - ------- - list[Path] - List of executable paths - - Raises - ------ - ProgramInstallationError - If version is not cached or bindir cannot be determined - """ - return _DEFAULT_MANAGER.select( - program=program, - version=version, - bindir=bindir, - verbose=verbose, - ) - - def uninstall_program( program: str, version: str | None = None, @@ -1814,42 +1630,6 @@ def uninstall_program( ) -def get_executable( - program: str, - version: str | None = None, - bindir: str | Path | None = None, -) -> Path: - """ - Get path to installed executable. - - Convenience wrapper for ProgramManager.get_executable(). - - Parameters - ---------- - program : str - Program name - version : str, optional - Program version (default: active version in bindir) - bindir : str | Path, optional - Installation directory (default: search in priority order) - - Returns - ------- - Path - Path to executable - - Raises - ------ - ProgramInstallationError - If executable not found - """ - return _DEFAULT_MANAGER.get_executable( - program=program, - version=version, - bindir=bindir, - ) - - def list_installed(program: str | None = None) -> dict[str, list[ProgramInstallation]]: """ List installed programs. @@ -1917,11 +1697,9 @@ def _try_best_effort_sync(): "download_archive", "extract_executables", "get_bindir_options", - "get_executable", "get_platform", "get_user_config_path", "install_program", "list_installed", - "select_version", "uninstall_program", ] diff --git a/modflow_devtools/programs/__main__.py b/modflow_devtools/programs/__main__.py index fe0d7f27..3de970aa 100644 --- a/modflow_devtools/programs/__main__.py +++ b/modflow_devtools/programs/__main__.py @@ -6,9 +6,8 @@ info Show sync status list List available programs install Install a program - select Switch active program version uninstall Uninstall a program - which Show path to installed executable + history Show installation history """ import argparse @@ -19,10 +18,8 @@ _DEFAULT_CACHE, ProgramSourceConfig, _try_best_effort_sync, - get_executable, install_program, list_installed, - select_version, uninstall_program, ) @@ -143,33 +140,6 @@ def cmd_install(args): sys.exit(1) -def cmd_select(args): - """Select command handler.""" - # Parse program@version format - if "@" in args.program: - program, version = args.program.split("@", 1) - else: - print( - "Error: Must specify version with program@version format", - file=sys.stderr, - ) - sys.exit(1) - - try: - paths = select_version( - program=program, - version=version, - bindir=args.bindir, - verbose=True, - ) - print("\nActivated executables:") - for path in paths: - print(f" {path}") - except Exception as e: - print(f"Selection failed: {e}", file=sys.stderr) - sys.exit(1) - - def cmd_uninstall(args): """Uninstall command handler.""" # Parse program@version format if provided @@ -200,21 +170,7 @@ def cmd_uninstall(args): sys.exit(1) -def cmd_which(args): - """Which command handler.""" - try: - path = get_executable( - program=args.program, - version=args.version, - bindir=args.bindir, - ) - print(path) - except Exception as e: - print(f"Executable not found: {e}", file=sys.stderr) - sys.exit(1) - - -def cmd_list_installed(args): +def cmd_history(args): """List installed programs command handler.""" installed = list_installed(args.program) @@ -225,12 +181,11 @@ def cmd_list_installed(args): print("No programs installed") return - print("Installed programs:\n") + print("Installation history:\n") for program_name, installations in sorted(installed.items()): print(f"{program_name}:") for inst in sorted(installations, key=lambda i: i.version): - active_marker = " (active)" if inst.active else "" - print(f" {inst.version} in {inst.bindir}{active_marker}") + print(f" {inst.version} in {inst.bindir}") if args.verbose: print(f" Platform: {inst.platform}") timestamp = inst.installed_at.strftime("%Y-%m-%d %H:%M:%S") @@ -303,17 +258,6 @@ def main(): help="Force reinstallation", ) - # Select command - select_parser = subparsers.add_parser("select", help="Switch active program version") - select_parser.add_argument( - "program", - help="Program name with version (program@version)", - ) - select_parser.add_argument( - "--bindir", - help="Installation directory", - ) - # Uninstall command uninstall_parser = subparsers.add_parser("uninstall", help="Uninstall a program") uninstall_parser.add_argument( @@ -336,29 +280,14 @@ def main(): help="Also remove from cache", ) - # Which command - which_parser = subparsers.add_parser("which", help="Show path to installed executable") - which_parser.add_argument( - "program", - help="Program name", - ) - which_parser.add_argument( - "--version", - help="Program version", - ) - which_parser.add_argument( - "--bindir", - help="Installation directory", - ) - - # Installed command (list installed programs) - installed_parser = subparsers.add_parser("installed", help="List installed programs") - installed_parser.add_argument( + # History command (list installation history) + history_parser = subparsers.add_parser("history", help="Show installation history") + history_parser.add_argument( "program", nargs="?", help="Specific program to list (default: all)", ) - installed_parser.add_argument( + history_parser.add_argument( "-v", "--verbose", action="store_true", @@ -380,14 +309,10 @@ def main(): cmd_list(args) elif args.command == "install": cmd_install(args) - elif args.command == "select": - cmd_select(args) elif args.command == "uninstall": cmd_uninstall(args) - elif args.command == "which": - cmd_which(args) - elif args.command == "installed": - cmd_list_installed(args) + elif args.command == "history": + cmd_history(args) else: parser.print_help() sys.exit(1) diff --git a/modflow_devtools/programs/make_registry.py b/modflow_devtools/programs/make_registry.py index 3c391c22..b70541fe 100644 --- a/modflow_devtools/programs/make_registry.py +++ b/modflow_devtools/programs/make_registry.py @@ -11,7 +11,9 @@ import argparse import hashlib import sys +import tarfile import tempfile +import zipfile from glob import glob from pathlib import Path @@ -61,6 +63,77 @@ def download_asset(asset_url: str, output_path: Path) -> None: f.write(chunk) +def peek_archive_for_exe(archive_path: Path, program_name: str, platform: str) -> str | None: + """ + Peek inside archive to find executable path. + + Supports both nested (archive_name/bin/program) and flat (bin/program) patterns. + + Parameters + ---------- + archive_path : Path + Path to archive file + program_name : str + Name of program to find + platform : str + Platform name (for determining exe extension) + + Returns + ------- + str | None + Path to executable within archive, or None if not found + """ + # Determine expected executable name + is_windows = platform.startswith("win") + + # Generate possible executable names + possible_names = [] + if is_windows: + possible_names.extend( + [ + f"{program_name}.exe", + f"{program_name}.dll", # For libraries like libmf6 + ] + ) + else: + possible_names.extend( + [ + program_name, + f"{program_name}.so", # Linux shared libraries + f"{program_name}.dylib", # macOS shared libraries + f"lib{program_name}.so", # libmf6.so + f"lib{program_name}.dylib", # libmf6.dylib + ] + ) + + try: + # List archive contents + if archive_path.suffix.lower() == ".zip": + with zipfile.ZipFile(archive_path, "r") as zf: + members = zf.namelist() + elif archive_path.suffix.lower() in [".gz", ".tgz"]: + with tarfile.open(archive_path, "r:gz") as tf: + members = tf.getnames() + elif archive_path.suffix.lower() == ".tar": + with tarfile.open(archive_path, "r") as tf: + members = tf.getnames() + else: + return None + + # Search for executable in priority order (executables before libraries) + for name in possible_names: + for member in members: + member_path = Path(member) + if member_path.name == name: + # Found it! Return the path + return member.replace("\\", "/") # Normalize to forward slashes + + return None + + except (zipfile.BadZipFile, tarfile.TarError): + return None + + def main(): parser = argparse.ArgumentParser( description="Generate a programs.toml registry file for a program release.", @@ -222,6 +295,7 @@ def main(): } temp_dir = None + downloaded_assets = {} # Cache: asset_name -> Path if args.compute_hashes: temp_dir = Path(tempfile.mkdtemp(prefix="programs-registry-")) @@ -245,6 +319,7 @@ def main(): # Find distributions for this program dists = [] + dist_exe_paths = {} # Track exe path for each dist (for pattern detection) for asset in assets: asset_name = asset["name"] @@ -275,31 +350,141 @@ def main(): "asset": asset_name, } - # Compute hash if requested - if args.compute_hashes: - if args.verbose: - print(" Computing hash...") - - if args.dists: - # Local file - use local_path - asset_path = Path(asset["local_path"]) + # Get archive path (for exe detection and optional hash computation) + asset_path = None + if args.dists: + # Local file - always available + asset_path = Path(asset["local_path"]) + else: + # GitHub release - download if not already cached + if asset_name in downloaded_assets: + asset_path = downloaded_assets[asset_name] else: - # GitHub release - download first + # Always download to enable exe detection and pattern optimization if args.verbose: - print(" Downloading to compute hash...") + action = ( + "to compute hash and detect exe path" + if args.compute_hashes + else "to detect exe path" + ) + print(f" Downloading {action}...") asset_url = asset["browser_download_url"] asset_path = temp_dir / asset_name download_asset(asset_url, asset_path) + downloaded_assets[asset_name] = asset_path + # Compute hash if requested + if args.compute_hashes: + if args.verbose: + print(" Computing hash...") hash_value = compute_sha256(asset_path) dist["hash"] = f"sha256:{hash_value}" - if args.verbose: print(f" SHA256: {hash_value}") + # Peek inside archive to find exe path (always do this for pattern optimization) + if asset_path and asset_path.exists(): + exe_path = peek_archive_for_exe(asset_path, program_name, matched_dist) + if exe_path: + dist_exe_paths[matched_dist] = exe_path + if args.verbose: + print(f" Found exe: {exe_path}") + dists.append(dist) if dists: + # Optimize: check if all dists follow a consistent pattern + # Patterns: nested ({asset_stem}/path) or flat (path at archive root) + if dist_exe_paths and len(dist_exe_paths) == len(dists): + # We have exe paths for all distributions + # Check if they all follow the same pattern (nested or flat) + consistent_pattern = True + relative_paths = [] + is_nested = None # Will be set to True/False after checking first dist + + for dist in dists: + dist_name = dist["name"] + asset_name = dist["asset"] + asset_stem = Path(asset_name).stem # Remove .zip extension + + if dist_name in dist_exe_paths: + exe_path = dist_exe_paths[dist_name] + + # Check if exe_path starts with asset_stem/ (nested pattern) + if exe_path.startswith(f"{asset_stem}/"): + # Nested pattern for this dist + if is_nested is False: + # Inconsistent: previous dists were flat + consistent_pattern = False + break + is_nested = True + rel_path = exe_path[ + len(asset_stem) + 1 : + ] # Remove asset_stem/ prefix + relative_paths.append(rel_path) + else: + # Flat pattern for this dist (no nested folder) + if is_nested is True: + # Inconsistent: previous dists were nested + consistent_pattern = False + break + is_nested = False + # exe_path is already the relative path + relative_paths.append(exe_path) + else: + consistent_pattern = False + break + + # Are all relative paths the same, + # ignoring platform-specific extensions? + if consistent_pattern and relative_paths: + normalized_paths = set() + for rp in relative_paths: + normalized = rp + for ext in [".exe", ".dll", ".so", ".dylib"]: + if normalized.endswith(ext): + normalized = normalized[: -len(ext)] + break + normalized_paths.add(normalized) + + if len(normalized_paths) == 1: + common_path = normalized_paths.pop() + + # Only store exe if it's not in a recognized location: + # - {program} + # - bin/{program} + if common_path not in [f"bin/{program_name}", program_name]: + program_meta["exe"] = common_path + if args.verbose: + pattern_type = "nested" if is_nested else "flat" + print(f" Detected {pattern_type} pattern") + else: + if args.verbose: + pattern_type = "nested" if is_nested else "flat" + print( + f" Detected {pattern_type} pattern with " + f"default path ({common_path})" + ) + else: + # Different relative paths, need dist-level exe entries + for dist in dists: + dist_name = dist["name"] + if dist_name in dist_exe_paths: + dist["exe"] = dist_exe_paths[dist_name] + else: + # Pattern not detected, use dist-level exe entries + for dist in dists: + dist_name = dist["name"] + if dist_name in dist_exe_paths: + dist["exe"] = dist_exe_paths[dist_name] + else: + # No exe paths found (archives may not be accessible) + if args.verbose: + print( + " Warning: Could not detect exe paths from archives. " + "Registry will use runtime detection." + ) + program_meta["dists"] = dists registry["programs"][program_name] = program_meta diff --git a/modflow_devtools/programs/programs.toml b/modflow_devtools/programs/programs.toml index 96666f6c..99370644 100644 --- a/modflow_devtools/programs/programs.toml +++ b/modflow_devtools/programs/programs.toml @@ -43,4 +43,12 @@ refs = ["v1.19.01"] [sources.swtv4] repo = "MODFLOW-ORG/swtv4" -refs = ["v4.00.05"] \ No newline at end of file +refs = ["v4.00.05"] + +[sources.mt3dms] +repo = "MODFLOW-ORG/mt3dms" +refs = ["v5.3.0"] + +[sources.mt3d-usgs] +repo = "MODFLOW-ORG/mt3d-usgs" +refs = ["v1.1.1"] \ No newline at end of file