Add python_requirements target generator

# Rust tests and lints will be skipped. Delete if not intended.
[ci skip-rust]

# Building wheels and fs_util will be skipped. Delete if not intended.
[ci skip-build-wheels]

src/python/pants/backend/python/macros/python_requirements.py

@@ -0,0 +1,147 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import itertools
+from typing import Iterable
+
+from packaging.utils import canonicalize_name as canonicalize_project_name
+
+from pants.backend.python.macros.common_fields import (
+    ModuleMappingField,
+    RequirementsOverrideField,
+    TypeStubsModuleMappingField,
+)
+from pants.backend.python.pip_requirement import PipRequirement
+from pants.backend.python.target_types import (
+    PythonRequirementModulesField,
+    PythonRequirementsField,
+    PythonRequirementsFileSourcesField,
+    PythonRequirementsFileTarget,
+    PythonRequirementTarget,
+    PythonRequirementTypeStubModulesField,
+    parse_requirements_file,
+)
+from pants.engine.addresses import Address
+from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs
+from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.target import (
+    COMMON_TARGET_FIELDS,
+    Dependencies,
+    GeneratedTargets,
+    GenerateTargetsRequest,
+    InvalidFieldException,
+    SingleSourceField,
+    Target,
+)
+from pants.util.logging import LogLevel
+
+
+class PythonRequirementsSourceField(SingleSourceField):
+    default = "requirements.txt"
+    required = False
+
+
+class PythonRequirementsTargetGenerator(Target):
+    alias = "python_requirements"
+    help = (
+        "Generate a `python_requirement` for each entry in a requirements.txt-style file.\n\n"
+        "This works with pip-style requirements files: "
+        "https://pip.pypa.io/en/latest/reference/requirements-file-format/. However, pip options "
+        "like `--hash` are (for now) ignored.\n\n"
+        "Instead of pip-style VCS requirements, use direct references from PEP 440: "
+        "https://www.python.org/dev/peps/pep-0440/#direct-references."
+    )
+    core_fields = (
+        *COMMON_TARGET_FIELDS,
+        ModuleMappingField,
+        TypeStubsModuleMappingField,
+        PythonRequirementsSourceField,
+        RequirementsOverrideField,
+    )
+
+
+class GenerateFromPythonRequirementsRequest(GenerateTargetsRequest):
+    generate_from = PythonRequirementsTargetGenerator
+
+
+@rule(desc="Generate `python_requirement` targets from requirements.txt", level=LogLevel.DEBUG)
+async def generate_from_python_requirement(
+    request: GenerateFromPythonRequirementsRequest,
+) -> GeneratedTargets:
+    generator = request.generator
+    requirements_rel_path = generator[PythonRequirementsSourceField].value
+    requirements_full_path = generator[PythonRequirementsSourceField].file_path
+
+    file_tgt = PythonRequirementsFileTarget(
+        {PythonRequirementsFileSourcesField.alias: requirements_rel_path},
+        Address(
+            generator.address.spec_path,
+            target_name=generator.address.target_name,
+            relative_file_path=requirements_rel_path,
+        ),
+    )
+
+    digest_contents = await Get(
+        DigestContents,
+        PathGlobs(
+            [requirements_full_path],
+            glob_match_error_behavior=GlobMatchErrorBehavior.error,
+            description_of_origin=f"{generator}'s field `{PythonRequirementsSourceField.alias}`",
+        ),
+    )
+    requirements = parse_requirements_file(
+        digest_contents[0].content.decode(), rel_path=requirements_full_path
+    )
+    grouped_requirements = itertools.groupby(
+        requirements, lambda parsed_req: parsed_req.project_name
+    )
+
+    module_mapping = generator[ModuleMappingField].value
+    stubs_mapping = generator[TypeStubsModuleMappingField].value
+    overrides = generator[RequirementsOverrideField].flatten_and_normalize()
+
+    def generate_tgt(
+        project_name: str, parsed_reqs: Iterable[PipRequirement]
+    ) -> PythonRequirementTarget:
+        normalized_proj_name = canonicalize_project_name(project_name)
+        tgt_overrides = overrides.pop(normalized_proj_name, {})
+        if Dependencies.alias in tgt_overrides:
+            tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [
+                file_tgt.address.spec
+            ]
+
+        # TODO: Consider letting you set metadata in the target generator and having it pass down
+        #  to all generated targets. Especially useful for compatible_resolves.
+        return PythonRequirementTarget(
+            {
+                PythonRequirementsField.alias: list(parsed_reqs),
+                PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name),
+                PythonRequirementTypeStubModulesField.alias: stubs_mapping.get(
+                    normalized_proj_name
+                ),
+                # This may get overridden by `tgt_overrides`, which will have already added in
+                # the file tgt.
+                Dependencies.alias: [file_tgt.address.spec],
+                **tgt_overrides,
+            },
+            generator.address.create_generated(project_name),
+        )
+
+    result = tuple(
+        generate_tgt(project_name, parsed_reqs_)
+        for project_name, parsed_reqs_ in grouped_requirements
+    ) + (file_tgt,)
+
+    if overrides:
+        raise InvalidFieldException(
+            f"Unused key in the `overrides` field for {request.generator.address}: "
+            f"{sorted(overrides)}"
+        )
+
+    return GeneratedTargets(generator, result)
+
+
+def rules():
+    return collect_rules()

src/python/pants/backend/python/macros/python_requirements_test.py

@@ -0,0 +1,197 @@
+# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from textwrap import dedent
+
+import pytest
+
+from pants.backend.python.macros import python_requirements
+from pants.backend.python.macros.python_requirements import (
+    GenerateFromPythonRequirementsRequest,
+    PythonRequirementsTargetGenerator,
+)
+from pants.backend.python.target_types import PythonRequirementsFileTarget, PythonRequirementTarget
+from pants.engine.addresses import Address
+from pants.engine.target import GeneratedTargets, Target
+from pants.testutil.rule_runner import QueryRule, RuleRunner, engine_error
+
+
+@pytest.fixture
+def rule_runner() -> RuleRunner:
+    return RuleRunner(
+        rules=[
+            *python_requirements.rules(),
+            QueryRule(GeneratedTargets, [GenerateFromPythonRequirementsRequest]),
+        ],
+        target_types=[PythonRequirementsTargetGenerator],
+    )
+
+
+def assert_python_requirements(
+    rule_runner: RuleRunner,
+    build_file_entry: str,
+    requirements_txt: str,
+    *,
+    expected_targets: set[Target],
+    requirements_txt_relpath: str = "requirements.txt",
+) -> None:
+    rule_runner.write_files({"BUILD": build_file_entry, requirements_txt_relpath: requirements_txt})
+    generator = rule_runner.get_target(Address("", target_name="reqs"))
+    result = rule_runner.request(
+        GeneratedTargets, [GenerateFromPythonRequirementsRequest(generator)]
+    )
+    assert set(result.values()) == expected_targets
+
+
+def test_requirements_txt(rule_runner: RuleRunner) -> None:
+    """This tests that we correctly create a new python_requirement for each entry in a
+    requirements.txt file, where each dependency is unique.
+
+    Some edge cases:
+    * We ignore comments and options (values that start with `--`).
+    * module_mapping works regardless of capitalization.
+    * Projects get normalized thanks to Requirement.parse().
+    * Overrides works, including for dependencies.
+    """
+    file_addr = Address("", target_name="reqs", relative_file_path="requirements.txt")
+    assert_python_requirements(
+        rule_runner,
+        dedent(
+            """\
+            python_requirements(
+                name='reqs',
+                module_mapping={'ansiCOLORS': ['colors']},
+                type_stubs_module_mapping={'Django-types': ['django']},
+                overrides={
+                  "ansicolors": {"tags": ["overridden"]},
+                  "Django": {"dependencies": ["#Django-types"]},
+                },
+            )
+            """
+        ),
+        dedent(
+            """\
+            # Comment.
+            --find-links=https://duckduckgo.com
+            ansicolors>=1.18.0
+            Django==3.2 ; python_version>'3'
+            Django-types
+            Un-Normalized-PROJECT  # Inline comment.
+            pip@ git+https://github.com/pypa/pip.git
+            """
+        ),
+        expected_targets={
+            PythonRequirementTarget(
+                {
+                    "requirements": ["ansicolors>=1.18.0"],
+                    "modules": ["colors"],
+                    "dependencies": [file_addr.spec],
+                    "tags": ["overridden"],
+                },
+                Address("", target_name="reqs", generated_name="ansicolors"),
+            ),
+            PythonRequirementTarget(
+                {
+                    "requirements": ["Django==3.2 ; python_version>'3'"],
+                    "dependencies": ["#Django-types", file_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="Django"),
+            ),
+            PythonRequirementTarget(
+                {
+                    "requirements": ["Django-types"],
+                    "type_stub_modules": ["django"],
+                    "dependencies": [file_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="Django-types"),
+            ),
+            PythonRequirementTarget(
+                {"requirements": ["Un_Normalized_PROJECT"], "dependencies": [file_addr.spec]},
+                Address("", target_name="reqs", generated_name="Un-Normalized-PROJECT"),
+            ),
+            PythonRequirementTarget(
+                {
+                    "requirements": ["pip@ git+https://github.com/pypa/pip.git"],
+                    "dependencies": [file_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="pip"),
+            ),
+            PythonRequirementsFileTarget({"source": "requirements.txt"}, file_addr),
+        },
+    )
+
+
+def test_multiple_versions(rule_runner: RuleRunner) -> None:
+    """This tests that we correctly create a new python_requirement for each unique dependency name
+    in a requirements.txt file, grouping duplicated dependency names to handle multiple requirement
+    strings per PEP 508."""
+    file_addr = Address("", target_name="reqs", relative_file_path="requirements.txt")
+    assert_python_requirements(
+        rule_runner,
+        "python_requirements(name='reqs')",
+        dedent(
+            """\
+            Django>=3.2
+            Django==3.2.7
+            confusedmonkey==86
+            repletewateringcan>=7
+            """
+        ),
+        expected_targets={
+            PythonRequirementTarget(
+                {
+                    "requirements": ["Django>=3.2", "Django==3.2.7"],
+                    "dependencies": [file_addr.spec],
+                },
+                Address("", target_name="reqs", generated_name="Django"),
+            ),
+            PythonRequirementTarget(
+                {"requirements": ["confusedmonkey==86"], "dependencies": [file_addr.spec]},
+                Address("", target_name="reqs", generated_name="confusedmonkey"),
+            ),
+            PythonRequirementTarget(
+                {"requirements": ["repletewateringcan>=7"], "dependencies": [file_addr.spec]},
+                Address("", target_name="reqs", generated_name="repletewateringcan"),
+            ),
+            PythonRequirementsFileTarget({"source": "requirements.txt"}, file_addr),
+        },
+    )
+
+
+def test_invalid_req(rule_runner: RuleRunner) -> None:
+    """Test that we give a nice error message."""
+    with engine_error(
+        contains="Invalid requirement 'Not A Valid Req == 3.7' in requirements.txt at line 3"
+    ):
+        assert_python_requirements(
+            rule_runner,
+            "python_requirements(name='reqs')",
+            "\n\nNot A Valid Req == 3.7",
+            expected_targets=set(),
+        )
+
+    # Give a nice error message if it looks like they're using pip VCS-style requirements.
+    with engine_error(contains="It looks like you're trying to use a pip VCS-style requirement?"):
+        assert_python_requirements(
+            rule_runner,
+            "python_requirements(name='reqs')",
+            "git+https://github.com/pypa/pip.git#egg=pip",
+            expected_targets=set(),
+        )
+
+
+def test_source_override(rule_runner: RuleRunner) -> None:
+    file_addr = Address("", target_name="reqs", relative_file_path="subdir/requirements.txt")
+    assert_python_requirements(
+        rule_runner,
+        "python_requirements(name='reqs', source='subdir/requirements.txt')",
+        "ansicolors>=1.18.0",
+        requirements_txt_relpath="subdir/requirements.txt",
+        expected_targets={
+            PythonRequirementTarget(
+                {"requirements": ["ansicolors>=1.18.0"], "dependencies": [file_addr.spec]},
+                Address("", target_name="reqs", generated_name="ansicolors"),
+            ),
+            PythonRequirementsFileTarget({"source": "subdir/requirements.txt"}, file_addr),
+        },
+    )