Include dynamic build dependencies

README.rst

@@ -562,12 +562,12 @@ Continuing the ``pyproject.toml`` example from earlier creating a single lock fi
 
 .. code-block:: console
 
-    $pip-compile --all-extras --build-system-requires --output-file=constraints.txt --strip-extras pyproject.toml
+    $pip-compile --all-build-distributions --all-extras --output-file=constraints.txt --strip-extras pyproject.toml
     #
     # This file is autogenerated by pip-compile with Python 3.9
     # by the following command:
     #
-    #    pip-compile --all-extras --build-system-requires --output-file=constraints.txt --strip-extras pyproject.toml
+    #    pip-compile --all-build-distributions --all-extras --output-file=constraints.txt --strip-extras pyproject.toml
     #
     asgiref==3.5.2
         # via django

examples/readme/constraints.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
-#    pip-compile --all-extras --build-system-requires --output-file=constraints.txt --strip-extras pyproject.toml
+#    pip-compile --all-build-distributions --all-extras --output-file=constraints.txt --strip-extras pyproject.toml
 #
 asgiref==3.5.2
     # via django

piptools/scripts/compile.py

@@ -5,7 +5,7 @@
 import shlex
 import sys
 import tempfile
-from typing import IO, Any, BinaryIO, cast
+from typing import IO, Any, BinaryIO, Iterable, cast
 
 import click
 import pep517
@@ -34,17 +34,22 @@
 )
 from ..writer import OutputWriter
 
+ALL_BUILD_DISTRIBUTIONS = ("editable", "sdist", "wheel")
 DEFAULT_REQUIREMENTS_FILE = "requirements.in"
 DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt"
 METADATA_FILENAMES = frozenset({"setup.py", "setup.cfg", "pyproject.toml"})
 
 
-def _build_system_requires(src_dir: str) -> set[str]:
-    result = ProjectBuilder(
-        src_dir, runner=pep517.quiet_subprocess_runner
-    ).build_system_requires
-    assert isinstance(result, set)
-    return result
+def _build_requirements(src_dir: str, distributions: Iterable[str]) -> set[str]:
+    builder = ProjectBuilder(src_dir, runner=pep517.quiet_subprocess_runner)
+    # It is not clear that it should be possible to use `get_requires_for_build` with
+    # "editable" but it seems to work in practice.
+    return set(
+        itertools.chain(
+            builder.build_system_requires,
+            *[builder.get_requires_for_build(dist) for dist in distributions],
+        )
+    )
 
 
 def _get_default_option(option_name: str) -> Any:
@@ -301,10 +306,19 @@ def _determine_linesep(
     f"Replaces default unsafe packages: {', '.join(sorted(UNSAFE_PACKAGES))}",
 )
 @click.option(
-    "--build-system-requires/--no-build-system-requires",
+    "--build-distribution",
+    "build_distributions",
+    multiple=True,
+    type=click.Choice(("editable", "sdist", "wheel")),
+    help="Name of a distribution to install build dependencies for; may be used more than once. "
+    "Static dependencies declared in pyproject.toml will be included as well.",
+)
+@click.option(
+    "--all-build-distributions",
     is_flag=True,
     default=False,
-    help="Pin also build requirements",
+    help="Pin build dependencies requested by any of the build backend hooks. "
+    "Static dependencies declared in pyproject.toml will be included as well.",
 )
 def cli(
     ctx: click.Context,
@@ -343,7 +357,8 @@ def cli(
     emit_index_url: bool,
     emit_options: bool,
     unsafe_package: tuple[str, ...],
-    build_system_requires: bool,
+    build_distributions: tuple[str, ...],
+    all_build_distributions: bool,
 ) -> None:
     """
     Compiles requirements.txt from requirements.in, pyproject.toml, setup.cfg,
@@ -494,7 +509,10 @@ def cli(
             setup_file_found = True
             try:
                 src_dir = os.path.dirname(os.path.abspath(src_file))
-                metadata = project_wheel_metadata(src_dir)
+                metadata = project_wheel_metadata(
+                    src_dir,
+                    isolated=build_isolation,
+                )
             except BuildBackendException as e:
                 log.error(str(e))
                 log.error(f"Failed to parse {os.path.abspath(src_file)}")
@@ -511,11 +529,19 @@ def cli(
                     msg = "--extra has no effect when used with --all-extras"
                     raise click.BadParameter(msg)
                 extras = tuple(metadata.get_all("Provides-Extra"))
-            if build_system_requires:
+            if all_build_distributions:
+                if build_distributions:
+                    msg = (
+                        "--build-distribution has no effect when used with "
+                        "--all-build-distributions"
+                    )
+                    raise click.BadParameter(msg)
+                build_distributions = ALL_BUILD_DISTRIBUTIONS
+            if build_distributions:
                 constraints.extend(
                     [
                         install_req_from_line(req, comes_from=comes_from)
-                        for req in _build_system_requires(src_dir)
+                        for req in _build_requirements(src_dir, build_distributions)
                     ]
                 )
         else:

tests/test_cli_compile.py

@@ -3,6 +3,7 @@
 import fnmatch
 import hashlib
 import os
+import pathlib
 import shutil
 import subprocess
 import sys
@@ -2496,32 +2497,32 @@ def test_all_extras_fail_with_extra(fake_dists, runner, make_module, fname, cont
     assert exp in out.stderr
 
 
-def _actual_deps(keys: Iterable[str], text: str) -> dict[str, str | None]:
-    # TODO: Consider making it possible to compare the actual- and expected-deps
-    #  without adding a bunch of None in the actual deps.
+def _parse_deps(text: str, keys: Iterable[str]) -> dict[str, str | None]:
+    """Parse `pip-compile` output and return mapping from package name to package version
+
+    :param text: output from `pip-compile`
+    :param keys: names of expected packages. If not in text the version will be set to `None`.
+    """
     result: dict[str, str | None] = {k: None for k in keys}
 
     for line in text.splitlines():
-        if line.startswith("#"):
+        prefix = line.split("#")[0].strip()
+        if not prefix:
             continue
         try:
-            k, v = line.split("==")
+            k, v = prefix.split("==")
         except ValueError:
             continue
         result[k] = v
 
-    print(result)
     return result
 
 
 class Version:
-    def __init__(self, expected: str | None) -> None:
+    def __init__(self, expected: str) -> None:
         self._pat = expected
 
     def __eq__(self, other: object) -> bool:
-        if None in {self._pat, other}:
-            return self._pat is other
-
         if not isinstance(other, str):
             return False
 
@@ -2531,76 +2532,144 @@ def __repr__(self) -> str:
         return f"<{self.__class__.__name__}@{id(self)}(_pat={self._pat!r})>"
 
 
-# TODO: Consider isolating test results from PyPi
-METADATA_TEST_CASE_BUILD_DEPS = {
-    "flit": {
-        "flit-core": Version("*"),
-        "poetry-core": Version(None),
-        "setuptools": Version(None),
-        "wheel": Version(None),
-    },
-    "poetry": {
-        "flit-core": Version(None),
-        "poetry-core": Version("*"),
-        "setuptools": Version(None),
-        "wheel": Version(None),
-    },
-    "setup.cfg": {
-        "flit-core": Version(None),
-        "poetry-core": Version(None),
-        "setuptools": Version("*"),
-        "wheel": Version(None),
-    },
-    "setup.py": {
-        "flit-core": Version(None),
-        "poetry-core": Version(None),
-        "setuptools": Version("*"),
-        "wheel": Version("*"),
-    },
-}
+# This can be removed when support for python<3.8 is dropped
+def copytree_dirs_exist_ok(
+    src_top: str | pathlib.Path, dst_top: str | pathlib.Path
+) -> None:
+    src_top = pathlib.Path(src_top)
+    dst_top = pathlib.Path(dst_top)
 
+    dst_top.mkdir(exist_ok=True)
+    for src in src_top.iterdir():
+        dst = dst_top / src.name
+        if src.is_file():
+            shutil.copy2(src, dst)
+        else:
+            shutil.copytree(src, dst)
 
+
+# TODO: Consider isolating from PyPI
 @pytest.mark.parametrize(
-    ("fname", "content", "build_deps"),
-    [
-        pytest.param(
-            *param.values, METADATA_TEST_CASE_BUILD_DEPS[param.id], id=param.id
-        )
-        for param in METADATA_TEST_CASES
-    ],
+    ("distributions", "expected_deps"),
+    (
+        (
+            ["editable"],
+            {
+                "setuptools": Version("*"),
+                "small-fake-c": Version("0.3"),
+            },
+        ),
+        (
+            ["sdist"],
+            {
+                "setuptools": Version("*"),
+                "small-fake-a": Version("0.1"),
+            },
+        ),
+        (
+            ["wheel"],
+            {
+                "setuptools": Version("*"),
+                "small-fake-b": Version("0.2"),
+                "wheel": Version("*"),
+            },
+        ),
+        (
+            ["editable", "sdist", "wheel"],
+            {
+                "setuptools": Version("*"),
+                "small-fake-a": Version("0.1"),
+                "small-fake-b": Version("0.2"),
+                "small-fake-c": Version("0.3"),
+                "wheel": Version("*"),
+            },
+        ),
+    ),
 )
-def test_build_system_requires(
-    fake_dists, runner, make_module, fname, content, build_deps
+def test_build_distribution(
+    fake_dists, runner, tmp_path, monkeypatch, distributions, expected_deps
+):
+    """
+    Test that when one or more --build-distribution are given the expected packages are included
+    """
+    # When used as argument to the runner it is not passed to pip
+    monkeypatch.setenv("PIP_FIND_LINKS", fake_dists)
+    src_pkg_path = os.path.join(PACKAGES_PATH, "small_fake_with_build_deps")
+    base_cmd = ["-n", "--allow-unsafe"]
+
+    with runner.isolated_filesystem(tmp_path) as tmp_pkg_path:
+        copytree_dirs_exist_ok(src_pkg_path, tmp_pkg_path)
+        out = runner.invoke(
+            cli,
+            base_cmd + [f"--build-distribution={d}" for d in distributions],
+        )
+
+    assert out.exit_code == 0
+    assert _parse_deps(out.stderr, expected_deps) == expected_deps
+
+
+def test_all_build_distributions(fake_dists, runner, tmp_path, monkeypatch):
+    """
+    Test that --all-build-distributions is equivalent to specifying every --build-distribution
+    """
+    # When used as argument to the runner it is not passed to pip
+    monkeypatch.setenv("PIP_FIND_LINKS", fake_dists)
+    src_pkg_path = os.path.join(PACKAGES_PATH, "small_fake_with_build_deps")
+    base_cmd = ["-n", "--allow-unsafe"]
+
+    with runner.isolated_filesystem(tmp_path) as tmp_pkg_path:
+        copytree_dirs_exist_ok(src_pkg_path, tmp_pkg_path)
+        actual_out = runner.invoke(
+            cli,
+            base_cmd + ["--all-build-distributions"],
+        )
+    actual_deps = _parse_deps(actual_out.stderr, [])
+
+    with runner.isolated_filesystem(tmp_path) as tmp_pkg_path:
+        copytree_dirs_exist_ok(src_pkg_path, tmp_pkg_path)
+        expected_out = runner.invoke(
+            cli,
+            base_cmd
+            + [
+                "--build-distribution=editable",
+                "--build-distribution=sdist",
+                "--build-distribution=wheel",
+            ],
+        )
+    expected_deps = _parse_deps(expected_out.stderr, [])
+
+    assert actual_out.exit_code == expected_out.exit_code == 0
+    assert actual_deps == expected_deps
+
+
+# This should not depend on the metadata format so testing all cases is wasteful
+@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES[:1])
+def test_all_build_distributions_fail_with_build_distribution(
+    fake_dists, runner, make_module, fname, content
 ):
     """
-    Test that requirements needed to build a package are included
+    Test that passing `--all-build-distributions` and `--build-distribution` fails.
     """
     meta_path = make_module(fname=fname, content=content)
     out = runner.invoke(
         cli,
         [
             "-n",
+            "--all-build-distributions",
+            "--build-distribution",
+            "sdist",
             "--find-links",
             fake_dists,
-            "--allow-unsafe",
             "--no-annotate",
             "--no-emit-options",
             "--no-header",
-            "--build-system-requires",
+            "--no-build-isolation",
             meta_path,
         ],
     )
-    assert out.exit_code == 0, out.stderr
-    expected_deps = {
-        "small-fake-a": Version("0.1"),
-        "small-fake-b": Version(None),
-        "small-fake-c": Version(None),
-    }
-    expected_deps.update(build_deps)
-
-    actual_deps = _actual_deps(expected_deps.keys(), out.stderr)
-    # Compare as dicts to see all differences in pytest output
-    assert actual_deps == expected_deps
+    assert out.exit_code == 2
+    exp = "--build-distribution has on effect when used with --all-build-distributions"
+    assert exp in out.stderr
 
 
 def test_extras_fail_with_requirements_in(runner, tmpdir):

tests/test_data/packages/small_fake_with_build_deps/backend/backend.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import setuptools.build_meta
+
+build_wheel = setuptools.build_meta.build_wheel
+build_sdist = setuptools.build_meta.build_sdist
+
+
+def get_requires_for_build_sdist(config_settings=None):
+    result = setuptools.build_meta.get_requires_for_build_sdist(config_settings)
+    assert result == []
+    result.append("small-fake-a")
+    return result
+
+
+def get_requires_for_build_wheel(config_settings=None):
+    result = setuptools.build_meta.get_requires_for_build_wheel(config_settings)
+    assert result == ["wheel"]
+    result.append("small-fake-b")
+    return result
+
+
+def get_requires_for_build_editable(config_settings=None):
+    return ["small-fake-c"]