Merge pull request #1129 from tiran/multiple-constraint-files

feat: implement multiple constraint files

Author: LalatenduMohanty

docs/how-tos/bootstrap-constraints.rst

@@ -35,3 +35,21 @@ production packages.
 
 This will use the constraints in the ``constraints.txt`` file to build the
 production packages for ``my-package``.
+
+Multiple constraints and remote constraints
+-------------------------------------------
+
+.. versionchanged:: 0.84.0
+   The ``--constraints-file`` / ``-c`` option now supports an arbitrary
+   number of arguments.
+
+The ``--constraints-file`` argument can be supplied multiple times. Multiple
+occurrences of the same package are merged and validated. For example
+``egg>=1.0`` and ``egg!=1.1.2`` are combined into ``egg>=1.0,!=1.1.2``. An
+unsatisfiable combination like ``egg<1.0`` and ``egg>2.0`` is an error.
+
+Fromager can load constraints from `https://` URLs, too.
+
+.. code-block:: console
+
+   $ fromager -c constraints.txt -c local-constraints.txt -c https://company.example/security-constraints.txt bootstrap my-package

docs/spelling_wordlist.txt

@@ -84,6 +84,7 @@ tos
 traceback
 tracebacks
 txt
+unsatisfiable
 unshare
 url
 urls

src/fromager/__main__.py

@@ -133,8 +133,13 @@
 @click.option(
     "-c",
     "--constraints-file",
+    "constraints_files",
+    multiple=True,
     type=str,
-    help="location of the constraints file",
+    help=(
+        "location of the constraints files. Constraints are merged and "
+        "checked for conflicts. Supports local path and remote from https://"
+    ),
 )
 @click.option(
     "--cleanup/--no-cleanup",
@@ -177,7 +182,7 @@ def main(
     patches_dir: pathlib.Path,
     settings_file: pathlib.Path,
     settings_dir: pathlib.Path,
-    constraints_file: str,
+    constraints_files: tuple[str, ...],
     cleanup: bool,
     variant: str,
     jobs: int | None,
@@ -247,7 +252,7 @@ def main(
             logger.info(f"variant: {variant}")
             logger.info(f"patches dir: {patches_dir}")
             logger.info(f"maximum concurrent jobs: {jobs}")
-            logger.info(f"constraints file: {constraints_file}")
+            logger.info(f"constraints files: {', '.join(constraints_files)}")
             logger.info(f"network isolation: {network_isolation}")
             if build_wheel_server_url:
                 logger.info(f"external build wheel server: {build_wheel_server_url}")
@@ -267,7 +272,7 @@ def main(
             variant=variant,
             max_jobs=jobs,
         ),
-        constraints_file=constraints_file,
+        constraints_files=constraints_files,
         patches_dir=patches_dir,
         sdists_repo=sdists_repo,
         wheels_repo=wheels_repo,

src/fromager/constraints.py

@@ -1,5 +1,6 @@
 import logging
 import pathlib
+import typing
 from collections.abc import Generator
 
 from packaging.requirements import Requirement
@@ -24,6 +25,9 @@ def __init__(self) -> None:
     def __iter__(self) -> Generator[NormalizedName, None, None]:
         yield from self._data
 
+    def __bool__(self) -> bool:
+        return bool(self._data)
+
     def __len__(self) -> int:
         return len(self._data)
 
@@ -70,12 +74,20 @@ def add_constraint(self, unparsed: str) -> None:
         self._data[canon_name] = req
 
     def load_constraints_file(self, constraints_file: str | pathlib.Path) -> None:
-        """Load constraints from a constraints file"""
+        """Load constraints from a constraints file or URL"""
         logger.info("loading constraints from %s", constraints_file)
         content = requirements_file.parse_requirements_file(constraints_file)
         for line in content:
             self.add_constraint(line)
 
+    def dump_constraints(self, output: typing.TextIO) -> None:
+        """Dump combined constraints to a text stream"""
+        # sort by normalized name
+        for _, req in sorted(self._data.items()):
+            # write requirement without markers. They have been evaluated
+            # in add_constraint()
+            output.write(f"{req.name}{req.specifier}\n")
+
     def get_constraint(self, name: str) -> Requirement | None:
         return self._data.get(canonicalize_name(name))
 

src/fromager/context.py

@@ -19,7 +19,6 @@
     dependency_graph,
     external_commands,
     packagesettings,
-    request_session,
 )
 
 if typing.TYPE_CHECKING:
@@ -35,12 +34,13 @@
 class WorkContext:
     def __init__(
         self,
+        *,
         active_settings: packagesettings.Settings | None,
-        constraints_file: str | None,
         patches_dir: pathlib.Path,
         sdists_repo: pathlib.Path,
         wheels_repo: pathlib.Path,
         work_dir: pathlib.Path,
+        constraints_files: tuple[str, ...] = (),
         cleanup: bool = True,
         variant: str = "cpu",
         network_isolation: bool = False,
@@ -59,13 +59,6 @@ def __init__(
                 max_jobs=max_jobs,
             )
         self.settings = active_settings
-        self.input_constraints_uri: str | None
-        self.constraints = constraints.Constraints()
-        if constraints_file is not None:
-            self.input_constraints_uri = constraints_file
-            self.constraints.load_constraints_file(constraints_file)
-        else:
-            self.input_constraints_uri = None
         self.sdists_repo = pathlib.Path(sdists_repo).resolve()
         self.sdists_downloads = self.sdists_repo / "downloads"
         self.sdists_builds = self.sdists_repo / "builds"
@@ -76,6 +69,7 @@ def __init__(
         self.wheel_server_dir = self.wheels_repo / "simple"
         self.work_dir = pathlib.Path(work_dir).resolve()
         self.graph_file = self.work_dir / "graph.json"
+        self.merged_constraints = self.work_dir / "merged-constraints.txt"
         self.uv_cache = self.work_dir / "uv-cache"
         self.wheel_server_url = wheel_server_url
         self.logs_dir = self.work_dir / "logs"
@@ -86,10 +80,13 @@ def __init__(
         self.network_isolation = network_isolation
         self.settings_dir = settings_dir
 
-        self._constraints_filename = self.work_dir / "constraints.txt"
-
         self.dependency_graph = dependency_graph.DependencyGraph()
 
+        self.constraints = constraints.Constraints()
+        self.input_constraints_files = constraints_files
+        for constraints_file in self.input_constraints_files:
+            self.constraints.load_constraints_file(constraints_file)
+
         # storing metrics
         self.time_store: dict[str, dict[str, float]] = collections.defaultdict(
             dict[str, float]
@@ -135,19 +132,9 @@ def pip_wheel_server_args(self) -> list[str]:
 
     @property
     def pip_constraint_args(self) -> list[str]:
-        if not self.input_constraints_uri:
+        if not self.constraints:
             return []
-
-        if self.input_constraints_uri.startswith(("https://", "http://", "file://")):
-            path_to_constraints_file = self.work_dir / "input-constraints.txt"
-            if not path_to_constraints_file.exists():
-                response = request_session.session.get(self.input_constraints_uri)
-                path_to_constraints_file.write_text(response.text)
-        else:
-            path_to_constraints_file = pathlib.Path(self.input_constraints_uri)
-
-        path_to_constraints_file = path_to_constraints_file.absolute()
-        return ["--constraint", os.fspath(path_to_constraints_file)]
+        return ["--constraint", os.fspath(self.merged_constraints)]
 
     def uv_clean_cache(self, *reqs: Requirement) -> None:
         """Invalidate and clean uv cache for requirements
@@ -201,6 +188,25 @@ def setup(self) -> None:
             if not p.exists():
                 logger.debug("creating %s", p)
                 p.mkdir(parents=True)
+        self.write_constraints()
+
+    def write_constraints(self) -> None:
+        """Write combined constraints to disk"""
+        if self.constraints:
+            with self.merged_constraints.open("w", encoding="utf-8") as f:
+                f.write("# auto-generated constraints file\n")
+                for constraints_file in self.input_constraints_files:
+                    f.write(f"# {constraints_file}\n")
+                f.write("\n")
+                self.constraints.dump_constraints(f)
+            logger.debug(
+                "generated %s with content %s",
+                self.merged_constraints,
+                self.merged_constraints.read_text(),
+            )
+        else:
+            logger.debug("no constraints configured")
+            self.merged_constraints.unlink(missing_ok=True)
 
     def clean_build_dirs(
         self,

tests/conftest.py

@@ -47,7 +47,6 @@ def tmp_context(tmp_path: pathlib.Path) -> context.WorkContext:
     variant = "cpu"
     ctx = context.WorkContext(
         active_settings=None,
-        constraints_file=None,
         patches_dir=patches_dir,
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",
@@ -73,7 +72,6 @@ def testdata_context(
             variant=variant,
             max_jobs=None,
         ),
-        constraints_file=None,
         patches_dir=overrides / "patches",
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",
@@ -107,7 +105,6 @@ def make_sbom_ctx(
         settings._package_settings[ps.name] = ps
     return context.WorkContext(
         active_settings=settings,
-        constraints_file=None,
         patches_dir=tmp_path / "patches",
         sdists_repo=tmp_path / "sdists-repo",
         wheels_repo=tmp_path / "wheels-repo",

tests/test_cli.py

@@ -1,5 +1,6 @@
 import pathlib
 
+import requests_mock
 from click.testing import CliRunner
 
 from fromager.__main__ import main as fromager
@@ -96,6 +97,40 @@ def test_output_dir_overridden_by_explicit_flags(
     assert not (out / "sdists-repo").exists()
 
 
+def test_multiple_constraints_files(
+    tmp_path: pathlib.Path, cli_runner: CliRunner
+) -> None:
+    constraints1 = tmp_path / "constraints1.txt"
+    constraints1.write_text("foo==1.0\nbar!=2.1.1\n")
+    constraints2 = tmp_path / "constraints2.txt"
+    constraints2.write_text("bar>=2.0\n")
+
+    url = "https://fromager.test/remote-constraints.txt"
+
+    with requests_mock.Mocker() as r:
+        r.get(
+            url,
+            text="remote>=1.0\n",
+        )
+        result = cli_runner.invoke(
+            fromager,
+            [
+                "--verbose",
+                "-c",
+                str(constraints1),
+                "--constraints-file",
+                str(constraints2),
+                "--constraints-file",
+                url,
+                "lint",
+            ],
+        )
+    assert result.exit_code == 0, result.output
+    assert "foo==1.0" in result.output
+    assert "bar!=2.1.1,>=2.0" in result.output
+    assert "remote>=1.0" in result.output
+
+
 KNOWN_COMMANDS: set[str] = {
     "bootstrap",
     "bootstrap-parallel",

tests/test_constraints.py

@@ -1,7 +1,9 @@
+import io
 import pathlib
 from unittest import mock
 
 import pytest
+import requests_mock
 from packaging import markers
 from packaging.requirements import Requirement
 from packaging.version import Version
@@ -11,7 +13,10 @@
 
 def test_constraint_is_satisfied_by() -> None:
     c = Constraints()
+    assert not c
     c.add_constraint("foo<=1.1")
+    assert c
+    assert len(c) == 1
     assert c.is_satisfied_by("foo", Version("1.1"))
     assert c.is_satisfied_by("foo", Version("1.0"))
     assert c.is_satisfied_by("bar", Version("2.0"))
@@ -91,6 +96,22 @@ def test_add_constraint_conflict() -> None:
     assert len(c) == 4  # flit_core, foo, bar, and baz
 
 
+def test_dump_constraints() -> None:
+    c = Constraints()
+
+    out = io.StringIO()
+    c.dump_constraints(out)
+    assert out.getvalue() == ""
+
+    c.add_constraint("foo>=1.0")
+    c.add_constraint("foo<2.0")
+    c.add_constraint("bar==1.1")
+
+    out = io.StringIO()
+    c.dump_constraints(out)
+    assert out.getvalue() == "bar==1.1\nfoo<2.0,>=1.0\n"
+
+
 def test_allow_prerelease() -> None:
     c = Constraints()
     c.add_constraint("foo>=1.1")
@@ -117,6 +138,18 @@ def test_load_constraints_file(tmp_path: pathlib.Path) -> None:
     assert c.get_constraint("torch") == Requirement("torch==3.1.0")
 
 
+def test_load_constraints_url() -> None:
+    c = Constraints()
+    url = "https://fromager.test/remote-constraints.txt"
+    with requests_mock.Mocker() as r:
+        r.get(
+            url,
+            text="remote>=1.0\n",
+        )
+        c.load_constraints_file(url)
+    assert c.get_constraint("remote") == Requirement("remote>=1.0")
+
+
 def test_invalid_constraints() -> None:
     c = Constraints()
     with pytest.raises(InvalidConstraintError, match=r".*no specifier"):