feat(bootstrap): add --multiple-versions flag to bootstrap all matching versions

Add --multiple-versions flag to bootstrap command that enables bootstrapping
all versions matching a requirement specification, rather than only the highest
version. This is useful for creating comprehensive build environments with
multiple versions of the same package.

Key changes:
- Add return_all_versions parameter to BootstrapRequirementResolver.resolve()
  with type-safe overloads using @overload decorators
- Modify Bootstrapper.bootstrap() to iterate over all resolved versions when
  --multiple-versions flag is enabled
- Implement continue-on-error behavior: failed versions are logged, tracked,
  and reported at the end without stopping the bootstrap process
- Add DependencyGraph.remove_dependency() to clean up failed nodes from graph
- Apply recursively to entire dependency chain (not just top-level)

Testing:
- Add 4 unit tests for resolver return_all_versions behavior
- Add unit test for continue-on-error and graph cleanup behavior
- Add e2e test using tomli>=2.0,<2.1 with constraints to verify multiple
  versions are bootstrapped successfully

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: rd4398

e2e/ci_bootstrap_suite.sh

@@ -25,6 +25,7 @@ test_section "bootstrap configuration tests"
 run_test "bootstrap_prerelease"
 run_test "bootstrap_cache"
 run_test "bootstrap_sdist_only"
+run_test "bootstrap_multiple_versions"
 
 test_section "bootstrap git URL tests"
 run_test "bootstrap_git_url"

e2e/test_bootstrap_multiple_versions.sh

@@ -0,0 +1,74 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test bootstrap with --multiple-versions flag
+# Tests that multiple matching versions are bootstrapped
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+
+# Create constraints file to pin build dependencies (keeps CI fast)
+constraints_file=$(mktemp)
+trap "rm -f $constraints_file" EXIT
+cat > "$constraints_file" <<EOF
+flit-core==3.11.0
+EOF
+
+# Use tomli with a version range that matches 3 versions (2.0.0, 2.0.1, 2.0.2)
+# tomli has no runtime dependencies, making it fast to bootstrap
+# It uses flit-core as build backend (pinned above)
+# Note: constraints file generation will fail (expected with multiple versions)
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --error-log-file="$OUTDIR/fromager-errors.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --constraints-file="$constraints_file" \
+  bootstrap \
+  --multiple-versions \
+  'tomli>=2.0,<2.1' || true
+
+# Check that wheels were built
+echo "Checking for wheels..."
+find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' | sort
+
+# Verify that we have multiple tomli wheels (only count downloads, not simple server copies)
+TOMLI_WHEEL_COUNT=$(find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl' | wc -l)
+echo "Found $TOMLI_WHEEL_COUNT tomli wheel(s)"
+
+# We expect exactly 3 versions (2.0.0, 2.0.1, 2.0.2)
+if [ "$TOMLI_WHEEL_COUNT" -ne 3 ]; then
+  echo "ERROR: Expected exactly 3 tomli wheels (2.0.0, 2.0.1, 2.0.2), found $TOMLI_WHEEL_COUNT"
+  echo "The --multiple-versions flag should have bootstrapped all matching versions"
+  exit 1
+fi
+
+# Check that we have all 3 versions
+HAS_2_0_0=false
+HAS_2_0_1=false
+HAS_2_0_2=false
+
+if find "$OUTDIR/wheels-repo/downloads/" -name "tomli-2.0.0-*.whl" | grep -q .; then
+  echo "Found wheel for tomli 2.0.0"
+  HAS_2_0_0=true
+fi
+
+if find "$OUTDIR/wheels-repo/downloads/" -name "tomli-2.0.1-*.whl" | grep -q .; then
+  echo "Found wheel for tomli 2.0.1"
+  HAS_2_0_1=true
+fi
+
+if find "$OUTDIR/wheels-repo/downloads/" -name "tomli-2.0.2-*.whl" | grep -q .; then
+  echo "Found wheel for tomli 2.0.2"
+  HAS_2_0_2=true
+fi
+
+if [ "$HAS_2_0_0" = false ] || [ "$HAS_2_0_1" = false ] || [ "$HAS_2_0_2" = false ]; then
+  echo "ERROR: Missing expected versions. Expected all of 2.0.0, 2.0.1, and 2.0.2"
+  echo "Found wheels:"
+  find "$OUTDIR/wheels-repo/downloads/" -name 'tomli-*.whl'
+  exit 1
+fi
+
+echo "SUCCESS: Multiple versions of tomli (2.0.0, 2.0.1, 2.0.2) were bootstrapped as expected"

src/fromager/bootstrap_requirement_resolver.py

@@ -8,6 +8,7 @@
 
 import logging
 import typing
+from typing import overload
 
 from packaging.requirements import Requirement
 from packaging.version import Version
@@ -56,13 +57,34 @@ def __init__(
             tuple[str, bool], list[tuple[str, Version]]
         ] = {}
 
+    @overload
     def resolve(
         self,
         req: Requirement,
         req_type: RequirementType,
         parent_req: Requirement | None = None,
         pre_built: bool | None = None,
-    ) -> tuple[str, Version]:
+        return_all_versions: typing.Literal[False] = False,
+    ) -> tuple[str, Version]: ...
+
+    @overload
+    def resolve(
+        self,
+        req: Requirement,
+        req_type: RequirementType,
+        parent_req: Requirement | None = None,
+        pre_built: bool | None = None,
+        return_all_versions: typing.Literal[True] = ...,
+    ) -> list[tuple[str, Version]]: ...
+
+    def resolve(
+        self,
+        req: Requirement,
+        req_type: RequirementType,
+        parent_req: Requirement | None = None,
+        pre_built: bool | None = None,
+        return_all_versions: bool = False,
+    ) -> tuple[str, Version] | list[tuple[str, Version]]:
         """Resolve package requirement to the best matching version.
 
         Tries resolution strategies in order:
@@ -76,9 +98,14 @@ def resolve(
             parent_req: Parent requirement from dependency chain
             pre_built: Optional override to force prebuilt (True) or source (False).
                 If None (default), uses package build info to determine.
+            return_all_versions: If True, return all matching versions instead of
+                just the highest version.
 
         Returns:
-            (url, version) tuple for the highest matching version
+            If return_all_versions is False (default):
+                (url, version) tuple for the highest matching version
+            If return_all_versions is True:
+                List of (url, version) tuples sorted by version (highest first)
 
         Raises:
             ValueError: If req contains a git URL and pre_built is False
@@ -101,14 +128,14 @@ def resolve(
         cached_result = self.get_cached_resolution(req, pre_built)
         if cached_result is not None:
             logger.debug(f"resolved {req} from cache")
-            return cached_result[0]
+            return cached_result if return_all_versions else cached_result[0]
 
         # Resolve using strategies
         results = self._resolve(req, req_type, parent_req, pre_built)
 
         # Cache the result
         self.cache_resolution(req, pre_built, results)
-        return results[0]
+        return results if return_all_versions else results[0]
 
     def _resolve(
         self,

src/fromager/bootstrapper.py

@@ -89,6 +89,7 @@ def __init__(
         cache_wheel_server_url: str | None = None,
         sdist_only: bool = False,
         test_mode: bool = False,
+        multiple_versions: bool = False,
     ) -> None:
         if test_mode and sdist_only:
             raise ValueError(
@@ -101,6 +102,7 @@ def __init__(
         self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url
         self.sdist_only = sdist_only
         self.test_mode = test_mode
+        self.multiple_versions = multiple_versions
         self.why: list[tuple[RequirementType, Requirement, Version]] = []
 
         # Delegate resolution to BootstrapRequirementResolver
@@ -126,6 +128,10 @@ def __init__(
         # Track failed packages in test mode (list of typed dicts for JSON export)
         self.failed_packages: list[FailureRecord] = []
 
+        # Track failed versions in multiple_versions mode
+        # Maps (package_name, version) -> exception info
+        self._failed_versions: list[tuple[str, str, Exception]] = []
+
     def resolve_and_add_top_level(
         self,
         req: Requirement,
@@ -249,22 +255,85 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None:
 
         In test mode, catches build exceptions, records package name, and continues.
         In normal mode, raises exceptions immediately (fail-fast).
+
+        When multiple_versions is enabled, bootstraps all matching versions instead
+        of just the highest version.
         """
         logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}")
 
-        # Resolve version first so we have it for error reporting.
+        # Resolve versions - get all if multiple_versions mode is enabled
         # In test mode, record resolution failures and continue.
+        resolved_versions: list[tuple[str, Version]]
         try:
-            source_url, resolved_version = self.resolve_version(
-                req=req,
-                req_type=req_type,
-            )
+            if self.multiple_versions:
+                # Get all matching versions
+                resolved_versions = self._resolver.resolve(
+                    req=req,
+                    req_type=req_type,
+                    parent_req=self.why[-1][1] if self.why else None,
+                    return_all_versions=True,
+                )
+                logger.info(
+                    f"multiple versions mode: resolved {len(resolved_versions)} version(s) for {req}"
+                )
+            else:
+                # Get only the highest version (default behavior)
+                source_url, resolved_version = self.resolve_version(
+                    req=req,
+                    req_type=req_type,
+                )
+                resolved_versions = [(source_url, resolved_version)]
         except Exception as err:
             if not self.test_mode:
                 raise
             self._record_test_mode_failure(req, None, err, "resolution")
             return
 
+        # Bootstrap each resolved version
+        for source_url, resolved_version in resolved_versions:
+            # For top-level requirements, add to graph before bootstrapping
+            # so that build dependencies can reference the parent node
+            if req_type == RequirementType.TOP_LEVEL:
+                pbi = self.ctx.package_build_info(req)
+                self.ctx.dependency_graph.add_dependency(
+                    parent_name=None,
+                    parent_version=None,
+                    req_type=RequirementType.TOP_LEVEL,
+                    req=req,
+                    req_version=resolved_version,
+                    download_url=source_url,
+                    pre_built=pbi.pre_built,
+                    constraint=self.ctx.constraints.get_constraint(req.name),
+                )
+                self.ctx.write_to_graph_to_file()
+
+            self._bootstrap_single_version(req, req_type, source_url, resolved_version)
+
+        # In multiple versions mode, report any failures for this requirement
+        if self.multiple_versions and self._failed_versions:
+            failed_for_req = [
+                (name, ver, exc)
+                for name, ver, exc in self._failed_versions
+                if name == canonicalize_name(req.name)
+            ]
+            if failed_for_req:
+                logger.warning(
+                    f"{req.name}: {len(failed_for_req)} version(s) failed to bootstrap"
+                )
+                for name, ver, exc in failed_for_req:
+                    logger.warning(f"  - {name}=={ver}: {type(exc).__name__}: {exc}")
+
+    def _bootstrap_single_version(
+        self,
+        req: Requirement,
+        req_type: RequirementType,
+        source_url: str,
+        resolved_version: Version,
+    ) -> None:
+        """Bootstrap a single version of a package.
+
+        Extracted from bootstrap() to handle both single and multiple version modes.
+        """
         # Capture parent before _track_why pushes current package onto the stack
         parent: tuple[Requirement, Version] | None = None
         if self.why:
@@ -298,11 +367,29 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> None:
                     req, req_type, source_url, resolved_version, build_sdist_only
                 )
             except Exception as err:
-                if not self.test_mode:
-                    raise
-                self._record_test_mode_failure(
-                    req, str(resolved_version), err, "bootstrap"
-                )
+                # In test_mode, record failure and continue
+                if self.test_mode:
+                    self._record_test_mode_failure(
+                        req, str(resolved_version), err, "bootstrap"
+                    )
+                    return
+
+                # In multiple_versions mode, record failure and continue to next version
+                if self.multiple_versions:
+                    pkg_name = canonicalize_name(req.name)
+                    self._failed_versions.append((pkg_name, str(resolved_version), err))
+                    logger.warning(
+                        f"{req.name}=={resolved_version}: failed to bootstrap: {type(err).__name__}: {err}"
+                    )
+                    # Remove failed node from graph since bootstrap didn't complete
+                    self.ctx.dependency_graph.remove_dependency(
+                        pkg_name, resolved_version
+                    )
+                    self.ctx.write_to_graph_to_file()
+                    return
+
+                # Otherwise, raise the exception (fail-fast)
+                raise
 
     def _bootstrap_impl(
         self,

src/fromager/commands/bootstrap.py

@@ -103,6 +103,13 @@ def _get_requirements_from_args(
     default=False,
     help="Test mode: continue processing after failures, report failures at end",
 )
+@click.option(
+    "--multiple-versions",
+    "multiple_versions",
+    is_flag=True,
+    default=False,
+    help="Bootstrap all matching versions instead of only the highest version",
+)
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
 def bootstrap(
@@ -113,6 +120,7 @@ def bootstrap(
     sdist_only: bool,
     skip_constraints: bool,
     test_mode: bool,
+    multiple_versions: bool,
     toplevel: list[str],
 ) -> None:
     """Compute and build the dependencies of a set of requirements recursively
@@ -147,6 +155,11 @@ def bootstrap(
             "test mode enabled: will continue processing after failures and report at end"
         )
 
+    if multiple_versions:
+        logger.info(
+            "multiple versions mode enabled: will bootstrap all matching versions"
+        )
+
     pre_built = wkctx.settings.list_pre_built()
     if pre_built:
         logger.info("treating %s as pre-built wheels", sorted(pre_built))
@@ -161,6 +174,7 @@ def bootstrap(
             cache_wheel_server_url,
             sdist_only=sdist_only,
             test_mode=test_mode,
+            multiple_versions=multiple_versions,
         )
 
         # Pre-resolution phase: Resolve all top-level dependencies before recursive
@@ -463,6 +477,13 @@ def write_constraints_file(
     default=None,
     help="maximum number of parallel workers to run (default: unlimited)",
 )
+@click.option(
+    "--multiple-versions",
+    "multiple_versions",
+    is_flag=True,
+    default=False,
+    help="Bootstrap all matching versions instead of only the highest version",
+)
 @click.argument("toplevel", nargs=-1)
 @click.pass_obj
 @click.pass_context
@@ -476,6 +497,7 @@ def bootstrap_parallel(
     skip_constraints: bool,
     force: bool,
     max_workers: int | None,
+    multiple_versions: bool,
     toplevel: list[str],
 ) -> None:
     """Bootstrap and build-parallel
@@ -502,6 +524,7 @@ def bootstrap_parallel(
         cache_wheel_server_url=cache_wheel_server_url,
         sdist_only=True,
         skip_constraints=skip_constraints,
+        multiple_versions=multiple_versions,
         toplevel=toplevel,
     )