feat: implement multiple constraint files

Fromager now supports multiple constraints files with `-c` /
``--constraints-file`` argument. Multiple constraints are merged and
validated.

Example:
```console
$ fromager \
    -c constraints.txt \
    -c local-constraints.txt \
    -c https://company.example/security-constraints.txt \
    bootstrap ...
```

Local and remote constraints are loaded in `WorkContext.setup()` and
dumped into a new file `merged-constraints.txt` in `work-dir`. Some
internals of `WorkContext` have changed in an API-incompatible way:

- `constraints_file` argument is now
  `constraints_files: tuple[str, ...] = ()`
- `WorkContext` now only accepts keyword arguments
- `input_constraints_uri` is replaced by `input_constraints_files`

Fixes: #1096
Signed-off-by: Christian Heimes <cheimes@redhat.com>

Author: tiran

docs/how-tos/bootstrap-constraints.rst

@@ -35,3 +35,21 @@ production packages.
 
 This will use the constraints in the ``constraints.txt`` file to build the
 production packages for ``my-package``.
+
+Multiple constraints and remote constraints
+-------------------------------------------
+
+.. versionchanged:: 0.84.0
+   The ``--constraints-file`` / ``-c`` option now supports an arbitrary
+   number of arguments.
+
+The ``--constraints-file`` argument can be supplied multiple times. Multiple
+occurrences of the same package are merged and validated. For examples
+``egg>=1.0`` and ``egg!=1.1.2`` are combined into ``egg>=1.0,!=1.1.2``. An
+unsatisfiable combination like ``egg<1.0`` and ``egg>2.0`` is an error.
+
+Fromager can load constraints from `https://` URLs, too.
+
+.. code-block:: console
+
+   $ fromager -c constraints.txt -c local-constraints.txt -c https://company.example/security-constraints.txt bootstrap my-package

docs/spelling_wordlist.txt

@@ -84,6 +84,7 @@ tos
 traceback
 tracebacks
 txt
+unsatisfiable
 unshare
 url
 urls

src/fromager/__main__.py

@@ -133,8 +133,13 @@
 @click.option(
     "-c",
     "--constraints-file",
+    "constraints_files",
+    multiple=True,
     type=str,
-    help="location of the constraints file",
+    help=(
+        "location of the constraints files. Constraints are merged and "
+        "checked for conflicts. Supports local path and remote from https://"
+    ),
 )
 @click.option(
     "--cleanup/--no-cleanup",
@@ -177,7 +182,7 @@ def main(
     patches_dir: pathlib.Path,
     settings_file: pathlib.Path,
     settings_dir: pathlib.Path,
-    constraints_file: str,
+    constraints_files: tuple[str, ...],
     cleanup: bool,
     variant: str,
     jobs: int | None,
@@ -247,7 +252,7 @@ def main(
             logger.info(f"variant: {variant}")
             logger.info(f"patches dir: {patches_dir}")
             logger.info(f"maximum concurrent jobs: {jobs}")
-            logger.info(f"constraints file: {constraints_file}")
+            logger.info(f"constraints files: {', '.join(constraints_files)}")
             logger.info(f"network isolation: {network_isolation}")
             if build_wheel_server_url:
                 logger.info(f"external build wheel server: {build_wheel_server_url}")
@@ -267,7 +272,7 @@ def main(
             variant=variant,
             max_jobs=jobs,
         ),
-        constraints_file=constraints_file,
+        constraints_files=constraints_files,
         patches_dir=patches_dir,
         sdists_repo=sdists_repo,
         wheels_repo=wheels_repo,

src/fromager/constraints.py

@@ -1,5 +1,6 @@
 import logging
 import pathlib
+import typing
 from collections.abc import Generator
 
 from packaging.requirements import Requirement
@@ -24,6 +25,9 @@ def __init__(self) -> None:
     def __iter__(self) -> Generator[NormalizedName, None, None]:
         yield from self._data
 
+    def __bool__(self) -> bool:
+        return bool(self._data)
+
     def __len__(self) -> int:
         return len(self._data)
 
@@ -70,12 +74,19 @@ def add_constraint(self, unparsed: str) -> None:
         self._data[canon_name] = req
 
     def load_constraints_file(self, constraints_file: str | pathlib.Path) -> None:
-        """Load constraints from a constraints file"""
+        """Load constraints from a constraints file or URL"""
         logger.info("loading constraints from %s", constraints_file)
         content = requirements_file.parse_requirements_file(constraints_file)
         for line in content:
             self.add_constraint(line)
 
+    def dump_constraints(self, output: typing.TextIO) -> None:
+        # sort by normalized name
+        for _, req in sorted(self._data.items()):
+            # write requirement without markers. They have been evaluated
+            # in add_constraint()
+            output.write(f"{req.name}{req.specifier}\n")
+
     def get_constraint(self, name: str) -> Requirement | None:
         return self._data.get(canonicalize_name(name))
 

src/fromager/context.py

@@ -19,7 +19,6 @@
     dependency_graph,
     external_commands,
     packagesettings,
-    request_session,
 )
 
 if typing.TYPE_CHECKING:
@@ -35,12 +34,13 @@
 class WorkContext:
     def __init__(
         self,
+        *,
         active_settings: packagesettings.Settings | None,
-        constraints_file: str | None,
         patches_dir: pathlib.Path,
         sdists_repo: pathlib.Path,
         wheels_repo: pathlib.Path,
         work_dir: pathlib.Path,
+        constraints_files: tuple[str, ...] = (),
         cleanup: bool = True,
         variant: str = "cpu",
         network_isolation: bool = False,
@@ -59,13 +59,6 @@ def __init__(
                 max_jobs=max_jobs,
             )
         self.settings = active_settings
-        self.input_constraints_uri: str | None
-        self.constraints = constraints.Constraints()
-        if constraints_file is not None:
-            self.input_constraints_uri = constraints_file
-            self.constraints.load_constraints_file(constraints_file)
-        else:
-            self.input_constraints_uri = None
         self.sdists_repo = pathlib.Path(sdists_repo).resolve()
         self.sdists_downloads = self.sdists_repo / "downloads"
         self.sdists_builds = self.sdists_repo / "builds"
@@ -76,6 +69,7 @@ def __init__(
         self.wheel_server_dir = self.wheels_repo / "simple"
         self.work_dir = pathlib.Path(work_dir).resolve()
         self.graph_file = self.work_dir / "graph.json"
+        self.merged_constraints = self.work_dir / "merged-constraints.txt"
         self.uv_cache = self.work_dir / "uv-cache"
         self.wheel_server_url = wheel_server_url
         self.logs_dir = self.work_dir / "logs"
@@ -86,10 +80,13 @@ def __init__(
         self.network_isolation = network_isolation
         self.settings_dir = settings_dir
 
-        self._constraints_filename = self.work_dir / "constraints.txt"
-
         self.dependency_graph = dependency_graph.DependencyGraph()
 
+        self.constraints = constraints.Constraints()
+        self.input_constraints_files = constraints_files
+        for constraints_file in self.input_constraints_files:
+            self.constraints.load_constraints_file(constraints_file)
+
         # storing metrics
         self.time_store: dict[str, dict[str, float]] = collections.defaultdict(
             dict[str, float]
@@ -135,19 +132,9 @@ def pip_wheel_server_args(self) -> list[str]:
 
     @property
     def pip_constraint_args(self) -> list[str]:
-        if not self.input_constraints_uri:
+        if not self.constraints:
             return []
-
-        if self.input_constraints_uri.startswith(("https://", "http://", "file://")):
-            path_to_constraints_file = self.work_dir / "input-constraints.txt"
-            if not path_to_constraints_file.exists():
-                response = request_session.session.get(self.input_constraints_uri)
-                path_to_constraints_file.write_text(response.text)
-        else:
-            path_to_constraints_file = pathlib.Path(self.input_constraints_uri)
-
-        path_to_constraints_file = path_to_constraints_file.absolute()
-        return ["--constraint", os.fspath(path_to_constraints_file)]
+        return ["--constraint", os.fspath(self.merged_constraints)]
 
     def uv_clean_cache(self, *reqs: Requirement) -> None:
         """Invalidate and clean uv cache for requirements
@@ -202,6 +189,22 @@ def setup(self) -> None:
                 logger.debug("creating %s", p)
                 p.mkdir(parents=True)
 
+        if self.constraints:
+            with self.merged_constraints.open("w", encoding="utf-8") as f:
+                f.write("# auto-generated constraints file\n")
+                for constraints_file in self.input_constraints_files:
+                    f.write(f"# {constraints_file}\n")
+                f.write("\n")
+                self.constraints.dump_constraints(f)
+            logger.debug(
+                "generated %s with content %s",
+                self.merged_constraints,
+                self.merged_constraints.read_text(),
+            )
+        else:
+            logger.debug("no constraints configured")
+            self.merged_constraints.unlink(missing_ok=True)
+
     def clean_build_dirs(
         self,
         sdist_root_dir: pathlib.Path | None,

tests/conftest.py

@@ -47,7 +47,6 @@ def tmp_context(tmp_path: pathlib.Path) -> context.WorkContext:
     variant = "cpu"
     ctx = context.WorkContext(
         active_settings=None,
-        constraints_file=None,
         patches_dir=patches_dir,
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",
@@ -73,7 +72,6 @@ def testdata_context(
             variant=variant,
             max_jobs=None,
         ),
-        constraints_file=None,
         patches_dir=overrides / "patches",
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",
@@ -107,7 +105,6 @@ def make_sbom_ctx(
         settings._package_settings[ps.name] = ps
     return context.WorkContext(
         active_settings=settings,
-        constraints_file=None,
         patches_dir=tmp_path / "patches",
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",

tests/test_cli.py

@@ -96,6 +96,30 @@ def test_output_dir_overridden_by_explicit_flags(
     assert not (out / "sdists-repo").exists()
 
 
+def test_multiple_constraints_files(
+    tmp_path: pathlib.Path, cli_runner: CliRunner
+) -> None:
+    constraints1 = tmp_path / "constraints1.txt"
+    constraints1.write_text("foo==1.0\nbar!=2.1.1\n")
+    constraints2 = tmp_path / "constraints2.txt"
+    constraints2.write_text("bar>=2.0\n")
+
+    result = cli_runner.invoke(
+        fromager,
+        [
+            "--verbose",
+            "-c",
+            str(constraints1),
+            "--constraints-file",
+            str(constraints2),
+            "lint",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert "foo==1.0" in result.output
+    assert "bar!=2.1.1,>=2.0" in result.output
+
+
 KNOWN_COMMANDS: set[str] = {
     "bootstrap",
     "bootstrap-parallel",

tests/test_constraints.py

@@ -1,3 +1,4 @@
+import io
 import pathlib
 from unittest import mock
 
@@ -11,7 +12,10 @@
 
 def test_constraint_is_satisfied_by() -> None:
     c = Constraints()
+    assert not c
     c.add_constraint("foo<=1.1")
+    assert c
+    assert len(c) == 1
     assert c.is_satisfied_by("foo", Version("1.1"))
     assert c.is_satisfied_by("foo", Version("1.0"))
     assert c.is_satisfied_by("bar", Version("2.0"))
@@ -91,6 +95,22 @@ def test_add_constraint_conflict() -> None:
     assert len(c) == 4  # flit_core, foo, bar, and baz
 
 
+def test_dump_constraints() -> None:
+    c = Constraints()
+
+    out = io.StringIO()
+    c.dump_constraints(out)
+    assert out.getvalue() == ""
+
+    c.add_constraint("foo>=1.0")
+    c.add_constraint("foo<2.0")
+    c.add_constraint("bar==1.1")
+
+    out = io.StringIO()
+    c.dump_constraints(out)
+    assert out.getvalue() == "bar==1.1\nfoo<2.0,>=1.0\n"
+
+
 def test_allow_prerelease() -> None:
     c = Constraints()
     c.add_constraint("foo>=1.1")

tests/test_context.py

@@ -10,13 +10,13 @@
 
 def _make_context(
     tmp_path: pathlib.Path,
-    constraints_file: str | None = None,
+    constraints_files: tuple[str, ...] = (),
     wheel_server_url: str = "",
     cleanup: bool = True,
 ) -> context.WorkContext:
     return context.WorkContext(
         active_settings=None,
-        constraints_file=constraints_file,
+        constraints_files=constraints_files,
         patches_dir=tmp_path / "overrides/patches",
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",
@@ -43,16 +43,57 @@ def _all_setup_dirs(ctx: context.WorkContext) -> list[pathlib.Path]:
 
 def test_pip_constraints_args(tmp_path: pathlib.Path) -> None:
     constraints_file = tmp_path / "constraints.txt"
-    constraints_file.write_text("\n")  # the file has to exist
-    ctx = _make_context(tmp_path, constraints_file=str(constraints_file))
+    constraints_file.write_text("test==1.0\n")
+    ctx = _make_context(tmp_path, constraints_files=(str(constraints_file),))
     ctx.setup()
-    assert ["--constraint", os.fspath(constraints_file)] == ctx.pip_constraint_args
+    assert ctx.merged_constraints == tmp_path / "work-dir" / "merged-constraints.txt"
+    assert ctx.pip_constraint_args == [
+        "--constraint",
+        os.fspath(ctx.merged_constraints),
+    ]
+
+    assert ctx.merged_constraints.read_text() == "\n".join(
+        (
+            "# auto-generated constraints file",
+            f"# {constraints_file}",
+            "",
+            "test==1.0",
+            "",
+        )
+    )
 
     ctx = _make_context(tmp_path)
     ctx.setup()
     assert [] == ctx.pip_constraint_args
 
 
+def test_multiple_constraints_files(tmp_path: pathlib.Path) -> None:
+    constraints1 = tmp_path / "constraints1.txt"
+    constraints1.write_text("test==1.0\n")
+    constraints2 = tmp_path / "constraints2.txt"
+    constraints2.write_text("foo>=2.0\n")
+    constraints3 = tmp_path / "constraints3.txt"
+    constraints3.write_text("foo!=2.1.1\n")
+    ctx = _make_context(
+        tmp_path,
+        constraints_files=(str(constraints1), str(constraints2), str(constraints3)),
+    )
+    ctx.setup()
+
+    assert ctx.merged_constraints.read_text() == "\n".join(
+        (
+            "# auto-generated constraints file",
+            f"# {constraints1}",
+            f"# {constraints2}",
+            f"# {constraints3}",
+            "",
+            "foo!=2.1.1,>=2.0",
+            "test==1.0",
+            "",
+        )
+    )
+
+
 def test_setup_creates_directories(tmp_path: pathlib.Path) -> None:
     ctx = _make_context(tmp_path)
     ctx.setup()