diff --git a/docs/how-tos/bootstrap-constraints.rst b/docs/how-tos/bootstrap-constraints.rst index cdcacd541..6fcfb4eb1 100644 --- a/docs/how-tos/bootstrap-constraints.rst +++ b/docs/how-tos/bootstrap-constraints.rst @@ -35,3 +35,21 @@ production packages. This will use the constraints in the ``constraints.txt`` file to build the production packages for ``my-package``. + +Multiple constraints and remote constraints +------------------------------------------- + +.. versionchanged:: 0.84.0 + The ``--constraints-file`` / ``-c`` option now supports an arbitrary + number of arguments. + +The ``--constraints-file`` argument can be supplied multiple times. Multiple +occurrences of the same package are merged and validated. For example +``egg>=1.0`` and ``egg!=1.1.2`` are combined into ``egg>=1.0,!=1.1.2``. An +unsatisfiable combination like ``egg<1.0`` and ``egg>2.0`` is an error. + +Fromager can load constraints from `https://` URLs, too. + +.. code-block:: console + + $ fromager -c constraints.txt -c local-constraints.txt -c https://company.example/security-constraints.txt bootstrap my-package diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 9b42ab757..a00361eed 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -84,6 +84,7 @@ tos traceback tracebacks txt +unsatisfiable unshare url urls diff --git a/src/fromager/__main__.py b/src/fromager/__main__.py index 4738f8fae..2ac6f4b5f 100644 --- a/src/fromager/__main__.py +++ b/src/fromager/__main__.py @@ -133,8 +133,13 @@ @click.option( "-c", "--constraints-file", + "constraints_files", + multiple=True, type=str, - help="location of the constraints file", + help=( + "location of the constraints files. Constraints are merged and " + "checked for conflicts. Supports local path and remote from https://" + ), ) @click.option( "--cleanup/--no-cleanup", @@ -177,7 +182,7 @@ def main( patches_dir: pathlib.Path, settings_file: pathlib.Path, settings_dir: pathlib.Path, - constraints_file: str, + constraints_files: tuple[str, ...], cleanup: bool, variant: str, jobs: int | None, @@ -247,7 +252,7 @@ def main( logger.info(f"variant: {variant}") logger.info(f"patches dir: {patches_dir}") logger.info(f"maximum concurrent jobs: {jobs}") - logger.info(f"constraints file: {constraints_file}") + logger.info(f"constraints files: {', '.join(constraints_files)}") logger.info(f"network isolation: {network_isolation}") if build_wheel_server_url: logger.info(f"external build wheel server: {build_wheel_server_url}") @@ -267,7 +272,7 @@ def main( variant=variant, max_jobs=jobs, ), - constraints_file=constraints_file, + constraints_files=constraints_files, patches_dir=patches_dir, sdists_repo=sdists_repo, wheels_repo=wheels_repo, diff --git a/src/fromager/constraints.py b/src/fromager/constraints.py index 9ebc22d34..99b0e15fe 100644 --- a/src/fromager/constraints.py +++ b/src/fromager/constraints.py @@ -1,5 +1,6 @@ import logging import pathlib +import typing from collections.abc import Generator from packaging.requirements import Requirement @@ -24,6 +25,9 @@ def __init__(self) -> None: def __iter__(self) -> Generator[NormalizedName, None, None]: yield from self._data + def __bool__(self) -> bool: + return bool(self._data) + def __len__(self) -> int: return len(self._data) @@ -70,12 +74,20 @@ def add_constraint(self, unparsed: str) -> None: self._data[canon_name] = req def load_constraints_file(self, constraints_file: str | pathlib.Path) -> None: - """Load constraints from a constraints file""" + """Load constraints from a constraints file or URL""" logger.info("loading constraints from %s", constraints_file) content = requirements_file.parse_requirements_file(constraints_file) for line in content: self.add_constraint(line) + def dump_constraints(self, output: typing.TextIO) -> None: + """Dump combined constraints to a text stream""" + # sort by normalized name + for _, req in sorted(self._data.items()): + # write requirement without markers. They have been evaluated + # in add_constraint() + output.write(f"{req.name}{req.specifier}\n") + def get_constraint(self, name: str) -> Requirement | None: return self._data.get(canonicalize_name(name)) diff --git a/src/fromager/context.py b/src/fromager/context.py index 1bc543ed1..58866aedb 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -19,7 +19,6 @@ dependency_graph, external_commands, packagesettings, - request_session, ) if typing.TYPE_CHECKING: @@ -35,12 +34,13 @@ class WorkContext: def __init__( self, + *, active_settings: packagesettings.Settings | None, - constraints_file: str | None, patches_dir: pathlib.Path, sdists_repo: pathlib.Path, wheels_repo: pathlib.Path, work_dir: pathlib.Path, + constraints_files: tuple[str, ...] = (), cleanup: bool = True, variant: str = "cpu", network_isolation: bool = False, @@ -59,13 +59,6 @@ def __init__( max_jobs=max_jobs, ) self.settings = active_settings - self.input_constraints_uri: str | None - self.constraints = constraints.Constraints() - if constraints_file is not None: - self.input_constraints_uri = constraints_file - self.constraints.load_constraints_file(constraints_file) - else: - self.input_constraints_uri = None self.sdists_repo = pathlib.Path(sdists_repo).resolve() self.sdists_downloads = self.sdists_repo / "downloads" self.sdists_builds = self.sdists_repo / "builds" @@ -76,6 +69,7 @@ def __init__( self.wheel_server_dir = self.wheels_repo / "simple" self.work_dir = pathlib.Path(work_dir).resolve() self.graph_file = self.work_dir / "graph.json" + self.merged_constraints = self.work_dir / "merged-constraints.txt" self.uv_cache = self.work_dir / "uv-cache" self.wheel_server_url = wheel_server_url self.logs_dir = self.work_dir / "logs" @@ -86,10 +80,13 @@ def __init__( self.network_isolation = network_isolation self.settings_dir = settings_dir - self._constraints_filename = self.work_dir / "constraints.txt" - self.dependency_graph = dependency_graph.DependencyGraph() + self.constraints = constraints.Constraints() + self.input_constraints_files = constraints_files + for constraints_file in self.input_constraints_files: + self.constraints.load_constraints_file(constraints_file) + # storing metrics self.time_store: dict[str, dict[str, float]] = collections.defaultdict( dict[str, float] @@ -135,19 +132,9 @@ def pip_wheel_server_args(self) -> list[str]: @property def pip_constraint_args(self) -> list[str]: - if not self.input_constraints_uri: + if not self.constraints: return [] - - if self.input_constraints_uri.startswith(("https://", "http://", "file://")): - path_to_constraints_file = self.work_dir / "input-constraints.txt" - if not path_to_constraints_file.exists(): - response = request_session.session.get(self.input_constraints_uri) - path_to_constraints_file.write_text(response.text) - else: - path_to_constraints_file = pathlib.Path(self.input_constraints_uri) - - path_to_constraints_file = path_to_constraints_file.absolute() - return ["--constraint", os.fspath(path_to_constraints_file)] + return ["--constraint", os.fspath(self.merged_constraints)] def uv_clean_cache(self, *reqs: Requirement) -> None: """Invalidate and clean uv cache for requirements @@ -201,6 +188,25 @@ def setup(self) -> None: if not p.exists(): logger.debug("creating %s", p) p.mkdir(parents=True) + self.write_constraints() + + def write_constraints(self) -> None: + """Write combined constraints to disk""" + if self.constraints: + with self.merged_constraints.open("w", encoding="utf-8") as f: + f.write("# auto-generated constraints file\n") + for constraints_file in self.input_constraints_files: + f.write(f"# {constraints_file}\n") + f.write("\n") + self.constraints.dump_constraints(f) + logger.debug( + "generated %s with content %s", + self.merged_constraints, + self.merged_constraints.read_text(), + ) + else: + logger.debug("no constraints configured") + self.merged_constraints.unlink(missing_ok=True) def clean_build_dirs( self, diff --git a/tests/conftest.py b/tests/conftest.py index b99272611..495b0e9ad 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,7 +47,6 @@ def tmp_context(tmp_path: pathlib.Path) -> context.WorkContext: variant = "cpu" ctx = context.WorkContext( active_settings=None, - constraints_file=None, patches_dir=patches_dir, sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -73,7 +72,6 @@ def testdata_context( variant=variant, max_jobs=None, ), - constraints_file=None, patches_dir=overrides / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -107,7 +105,6 @@ def make_sbom_ctx( settings._package_settings[ps.name] = ps return context.WorkContext( active_settings=settings, - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", diff --git a/tests/test_cli.py b/tests/test_cli.py index d98845b75..85d97506f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,6 @@ import pathlib +import requests_mock from click.testing import CliRunner from fromager.__main__ import main as fromager @@ -96,6 +97,40 @@ def test_output_dir_overridden_by_explicit_flags( assert not (out / "sdists-repo").exists() +def test_multiple_constraints_files( + tmp_path: pathlib.Path, cli_runner: CliRunner +) -> None: + constraints1 = tmp_path / "constraints1.txt" + constraints1.write_text("foo==1.0\nbar!=2.1.1\n") + constraints2 = tmp_path / "constraints2.txt" + constraints2.write_text("bar>=2.0\n") + + url = "https://fromager.test/remote-constraints.txt" + + with requests_mock.Mocker() as r: + r.get( + url, + text="remote>=1.0\n", + ) + result = cli_runner.invoke( + fromager, + [ + "--verbose", + "-c", + str(constraints1), + "--constraints-file", + str(constraints2), + "--constraints-file", + url, + "lint", + ], + ) + assert result.exit_code == 0, result.output + assert "foo==1.0" in result.output + assert "bar!=2.1.1,>=2.0" in result.output + assert "remote>=1.0" in result.output + + KNOWN_COMMANDS: set[str] = { "bootstrap", "bootstrap-parallel", diff --git a/tests/test_constraints.py b/tests/test_constraints.py index e7acf8d18..219e1ce0f 100644 --- a/tests/test_constraints.py +++ b/tests/test_constraints.py @@ -1,7 +1,9 @@ +import io import pathlib from unittest import mock import pytest +import requests_mock from packaging import markers from packaging.requirements import Requirement from packaging.version import Version @@ -11,7 +13,10 @@ def test_constraint_is_satisfied_by() -> None: c = Constraints() + assert not c c.add_constraint("foo<=1.1") + assert c + assert len(c) == 1 assert c.is_satisfied_by("foo", Version("1.1")) assert c.is_satisfied_by("foo", Version("1.0")) assert c.is_satisfied_by("bar", Version("2.0")) @@ -91,6 +96,22 @@ def test_add_constraint_conflict() -> None: assert len(c) == 4 # flit_core, foo, bar, and baz +def test_dump_constraints() -> None: + c = Constraints() + + out = io.StringIO() + c.dump_constraints(out) + assert out.getvalue() == "" + + c.add_constraint("foo>=1.0") + c.add_constraint("foo<2.0") + c.add_constraint("bar==1.1") + + out = io.StringIO() + c.dump_constraints(out) + assert out.getvalue() == "bar==1.1\nfoo<2.0,>=1.0\n" + + def test_allow_prerelease() -> None: c = Constraints() c.add_constraint("foo>=1.1") @@ -117,6 +138,18 @@ def test_load_constraints_file(tmp_path: pathlib.Path) -> None: assert c.get_constraint("torch") == Requirement("torch==3.1.0") +def test_load_constraints_url() -> None: + c = Constraints() + url = "https://fromager.test/remote-constraints.txt" + with requests_mock.Mocker() as r: + r.get( + url, + text="remote>=1.0\n", + ) + c.load_constraints_file(url) + assert c.get_constraint("remote") == Requirement("remote>=1.0") + + def test_invalid_constraints() -> None: c = Constraints() with pytest.raises(InvalidConstraintError, match=r".*no specifier"): diff --git a/tests/test_context.py b/tests/test_context.py index c4b193396..948ca2a3c 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -10,13 +10,13 @@ def _make_context( tmp_path: pathlib.Path, - constraints_file: str | None = None, + constraints_files: tuple[str, ...] = (), wheel_server_url: str = "", cleanup: bool = True, ) -> context.WorkContext: return context.WorkContext( active_settings=None, - constraints_file=constraints_file, + constraints_files=constraints_files, patches_dir=tmp_path / "overrides/patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -43,16 +43,57 @@ def _all_setup_dirs(ctx: context.WorkContext) -> list[pathlib.Path]: def test_pip_constraints_args(tmp_path: pathlib.Path) -> None: constraints_file = tmp_path / "constraints.txt" - constraints_file.write_text("\n") # the file has to exist - ctx = _make_context(tmp_path, constraints_file=str(constraints_file)) + constraints_file.write_text("test==1.0\n") + ctx = _make_context(tmp_path, constraints_files=(str(constraints_file),)) ctx.setup() - assert ["--constraint", os.fspath(constraints_file)] == ctx.pip_constraint_args + assert ctx.merged_constraints == tmp_path / "work-dir" / "merged-constraints.txt" + assert ctx.pip_constraint_args == [ + "--constraint", + os.fspath(ctx.merged_constraints), + ] + + assert ctx.merged_constraints.read_text() == "\n".join( + ( + "# auto-generated constraints file", + f"# {constraints_file}", + "", + "test==1.0", + "", + ) + ) ctx = _make_context(tmp_path) ctx.setup() assert [] == ctx.pip_constraint_args +def test_multiple_constraints_files(tmp_path: pathlib.Path) -> None: + constraints1 = tmp_path / "constraints1.txt" + constraints1.write_text("test==1.0\n") + constraints2 = tmp_path / "constraints2.txt" + constraints2.write_text("foo>=2.0\n") + constraints3 = tmp_path / "constraints3.txt" + constraints3.write_text("foo!=2.1.1\n") + ctx = _make_context( + tmp_path, + constraints_files=(str(constraints1), str(constraints2), str(constraints3)), + ) + ctx.setup() + + assert ctx.merged_constraints.read_text() == "\n".join( + ( + "# auto-generated constraints file", + f"# {constraints1}", + f"# {constraints2}", + f"# {constraints3}", + "", + "foo!=2.1.1,>=2.0", + "test==1.0", + "", + ) + ) + + def test_setup_creates_directories(tmp_path: pathlib.Path) -> None: ctx = _make_context(tmp_path) ctx.setup() diff --git a/tests/test_cooldown.py b/tests/test_cooldown.py index aa27916d8..37c9ea063 100644 --- a/tests/test_cooldown.py +++ b/tests/test_cooldown.py @@ -223,7 +223,6 @@ def test_cooldown_applied_automatically_via_ctx(tmp_path: pathlib.Path) -> None: """ ctx = context.WorkContext( active_settings=None, - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -259,7 +258,6 @@ def test_toplevel_equality_pin_bypasses_cooldown_via_resolve( """ ctx = context.WorkContext( active_settings=None, - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -294,7 +292,6 @@ def test_cooldown_applied_via_get_source_provider(tmp_path: pathlib.Path) -> Non """ ctx = context.WorkContext( active_settings=None, - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -323,7 +320,6 @@ def test_non_pypi_index_allows_without_upload_time( """ ctx = context.WorkContext( active_settings=None, - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -382,7 +378,6 @@ def _make_ctx( variant="cpu", max_jobs=None, ), - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -451,7 +446,6 @@ def test_per_package_cooldown_disable_via_ctx(tmp_path: pathlib.Path) -> None: variant="cpu", max_jobs=None, ), - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo", @@ -674,7 +668,6 @@ def test_local_wheel_server_allows_without_upload_time( local_server_url = "http://127.0.0.1:9999/simple/" ctx = context.WorkContext( active_settings=None, - constraints_file=None, patches_dir=tmp_path / "patches", sdists_repo=tmp_path / "sdists-repo", wheels_repo=tmp_path / "wheels-repo",