From 66ba6f9c71a831665751945752dc593abd8a5f02 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Wed, 5 Jan 2022 11:15:18 -0700 Subject: [PATCH] Add `python_requirements` target generator # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../python/macros/python_requirements.py | 147 +++++++++++++ .../python/macros/python_requirements_test.py | 197 ++++++++++++++++++ 2 files changed, 344 insertions(+) create mode 100644 src/python/pants/backend/python/macros/python_requirements.py create mode 100644 src/python/pants/backend/python/macros/python_requirements_test.py diff --git a/src/python/pants/backend/python/macros/python_requirements.py b/src/python/pants/backend/python/macros/python_requirements.py new file mode 100644 index 000000000000..2f49484247c3 --- /dev/null +++ b/src/python/pants/backend/python/macros/python_requirements.py @@ -0,0 +1,147 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import itertools +from typing import Iterable + +from packaging.utils import canonicalize_name as canonicalize_project_name + +from pants.backend.python.macros.common_fields import ( + ModuleMappingField, + RequirementsOverrideField, + TypeStubsModuleMappingField, +) +from pants.backend.python.pip_requirement import PipRequirement +from pants.backend.python.target_types import ( + PythonRequirementModulesField, + PythonRequirementsField, + PythonRequirementsFileSourcesField, + PythonRequirementsFileTarget, + PythonRequirementTarget, + PythonRequirementTypeStubModulesField, + parse_requirements_file, +) +from pants.engine.addresses import Address +from pants.engine.fs import DigestContents, GlobMatchErrorBehavior, PathGlobs +from pants.engine.rules import Get, collect_rules, rule +from pants.engine.target import ( + COMMON_TARGET_FIELDS, + Dependencies, + GeneratedTargets, + GenerateTargetsRequest, + InvalidFieldException, + SingleSourceField, + Target, +) +from pants.util.logging import LogLevel + + +class PythonRequirementsSourceField(SingleSourceField): + default = "requirements.txt" + required = False + + +class PythonRequirementsTargetGenerator(Target): + alias = "python_requirements" + help = ( + "Generate a `python_requirement` for each entry in a requirements.txt-style file.\n\n" + "This works with pip-style requirements files: " + "https://pip.pypa.io/en/latest/reference/requirements-file-format/. However, pip options " + "like `--hash` are (for now) ignored.\n\n" + "Instead of pip-style VCS requirements, use direct references from PEP 440: " + "https://www.python.org/dev/peps/pep-0440/#direct-references." + ) + core_fields = ( + *COMMON_TARGET_FIELDS, + ModuleMappingField, + TypeStubsModuleMappingField, + PythonRequirementsSourceField, + RequirementsOverrideField, + ) + + +class GenerateFromPythonRequirementsRequest(GenerateTargetsRequest): + generate_from = PythonRequirementsTargetGenerator + + +@rule(desc="Generate `python_requirement` targets from requirements.txt", level=LogLevel.DEBUG) +async def generate_from_python_requirement( + request: GenerateFromPythonRequirementsRequest, +) -> GeneratedTargets: + generator = request.generator + requirements_rel_path = generator[PythonRequirementsSourceField].value + requirements_full_path = generator[PythonRequirementsSourceField].file_path + + file_tgt = PythonRequirementsFileTarget( + {PythonRequirementsFileSourcesField.alias: requirements_rel_path}, + Address( + generator.address.spec_path, + target_name=generator.address.target_name, + relative_file_path=requirements_rel_path, + ), + ) + + digest_contents = await Get( + DigestContents, + PathGlobs( + [requirements_full_path], + glob_match_error_behavior=GlobMatchErrorBehavior.error, + description_of_origin=f"{generator}'s field `{PythonRequirementsSourceField.alias}`", + ), + ) + requirements = parse_requirements_file( + digest_contents[0].content.decode(), rel_path=requirements_full_path + ) + grouped_requirements = itertools.groupby( + requirements, lambda parsed_req: parsed_req.project_name + ) + + module_mapping = generator[ModuleMappingField].value + stubs_mapping = generator[TypeStubsModuleMappingField].value + overrides = generator[RequirementsOverrideField].flatten_and_normalize() + + def generate_tgt( + project_name: str, parsed_reqs: Iterable[PipRequirement] + ) -> PythonRequirementTarget: + normalized_proj_name = canonicalize_project_name(project_name) + tgt_overrides = overrides.pop(normalized_proj_name, {}) + if Dependencies.alias in tgt_overrides: + tgt_overrides[Dependencies.alias] = list(tgt_overrides[Dependencies.alias]) + [ + file_tgt.address.spec + ] + + # TODO: Consider letting you set metadata in the target generator and having it pass down + # to all generated targets. Especially useful for compatible_resolves. + return PythonRequirementTarget( + { + PythonRequirementsField.alias: list(parsed_reqs), + PythonRequirementModulesField.alias: module_mapping.get(normalized_proj_name), + PythonRequirementTypeStubModulesField.alias: stubs_mapping.get( + normalized_proj_name + ), + # This may get overridden by `tgt_overrides`, which will have already added in + # the file tgt. + Dependencies.alias: [file_tgt.address.spec], + **tgt_overrides, + }, + generator.address.create_generated(project_name), + ) + + result = tuple( + generate_tgt(project_name, parsed_reqs_) + for project_name, parsed_reqs_ in grouped_requirements + ) + (file_tgt,) + + if overrides: + raise InvalidFieldException( + f"Unused key in the `overrides` field for {request.generator.address}: " + f"{sorted(overrides)}" + ) + + return GeneratedTargets(generator, result) + + +def rules(): + return collect_rules() diff --git a/src/python/pants/backend/python/macros/python_requirements_test.py b/src/python/pants/backend/python/macros/python_requirements_test.py new file mode 100644 index 000000000000..2eef075dde65 --- /dev/null +++ b/src/python/pants/backend/python/macros/python_requirements_test.py @@ -0,0 +1,197 @@ +# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from textwrap import dedent + +import pytest + +from pants.backend.python.macros import python_requirements +from pants.backend.python.macros.python_requirements import ( + GenerateFromPythonRequirementsRequest, + PythonRequirementsTargetGenerator, +) +from pants.backend.python.target_types import PythonRequirementsFileTarget, PythonRequirementTarget +from pants.engine.addresses import Address +from pants.engine.target import GeneratedTargets, Target +from pants.testutil.rule_runner import QueryRule, RuleRunner, engine_error + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[ + *python_requirements.rules(), + QueryRule(GeneratedTargets, [GenerateFromPythonRequirementsRequest]), + ], + target_types=[PythonRequirementsTargetGenerator], + ) + + +def assert_python_requirements( + rule_runner: RuleRunner, + build_file_entry: str, + requirements_txt: str, + *, + expected_targets: set[Target], + requirements_txt_relpath: str = "requirements.txt", +) -> None: + rule_runner.write_files({"BUILD": build_file_entry, requirements_txt_relpath: requirements_txt}) + generator = rule_runner.get_target(Address("", target_name="reqs")) + result = rule_runner.request( + GeneratedTargets, [GenerateFromPythonRequirementsRequest(generator)] + ) + assert set(result.values()) == expected_targets + + +def test_requirements_txt(rule_runner: RuleRunner) -> None: + """This tests that we correctly create a new python_requirement for each entry in a + requirements.txt file, where each dependency is unique. + + Some edge cases: + * We ignore comments and options (values that start with `--`). + * module_mapping works regardless of capitalization. + * Projects get normalized thanks to Requirement.parse(). + * Overrides works, including for dependencies. + """ + file_addr = Address("", target_name="reqs", relative_file_path="requirements.txt") + assert_python_requirements( + rule_runner, + dedent( + """\ + python_requirements( + name='reqs', + module_mapping={'ansiCOLORS': ['colors']}, + type_stubs_module_mapping={'Django-types': ['django']}, + overrides={ + "ansicolors": {"tags": ["overridden"]}, + "Django": {"dependencies": ["#Django-types"]}, + }, + ) + """ + ), + dedent( + """\ + # Comment. + --find-links=https://duckduckgo.com + ansicolors>=1.18.0 + Django==3.2 ; python_version>'3' + Django-types + Un-Normalized-PROJECT # Inline comment. + pip@ git+https://github.com/pypa/pip.git + """ + ), + expected_targets={ + PythonRequirementTarget( + { + "requirements": ["ansicolors>=1.18.0"], + "modules": ["colors"], + "dependencies": [file_addr.spec], + "tags": ["overridden"], + }, + Address("", target_name="reqs", generated_name="ansicolors"), + ), + PythonRequirementTarget( + { + "requirements": ["Django==3.2 ; python_version>'3'"], + "dependencies": ["#Django-types", file_addr.spec], + }, + Address("", target_name="reqs", generated_name="Django"), + ), + PythonRequirementTarget( + { + "requirements": ["Django-types"], + "type_stub_modules": ["django"], + "dependencies": [file_addr.spec], + }, + Address("", target_name="reqs", generated_name="Django-types"), + ), + PythonRequirementTarget( + {"requirements": ["Un_Normalized_PROJECT"], "dependencies": [file_addr.spec]}, + Address("", target_name="reqs", generated_name="Un-Normalized-PROJECT"), + ), + PythonRequirementTarget( + { + "requirements": ["pip@ git+https://github.com/pypa/pip.git"], + "dependencies": [file_addr.spec], + }, + Address("", target_name="reqs", generated_name="pip"), + ), + PythonRequirementsFileTarget({"source": "requirements.txt"}, file_addr), + }, + ) + + +def test_multiple_versions(rule_runner: RuleRunner) -> None: + """This tests that we correctly create a new python_requirement for each unique dependency name + in a requirements.txt file, grouping duplicated dependency names to handle multiple requirement + strings per PEP 508.""" + file_addr = Address("", target_name="reqs", relative_file_path="requirements.txt") + assert_python_requirements( + rule_runner, + "python_requirements(name='reqs')", + dedent( + """\ + Django>=3.2 + Django==3.2.7 + confusedmonkey==86 + repletewateringcan>=7 + """ + ), + expected_targets={ + PythonRequirementTarget( + { + "requirements": ["Django>=3.2", "Django==3.2.7"], + "dependencies": [file_addr.spec], + }, + Address("", target_name="reqs", generated_name="Django"), + ), + PythonRequirementTarget( + {"requirements": ["confusedmonkey==86"], "dependencies": [file_addr.spec]}, + Address("", target_name="reqs", generated_name="confusedmonkey"), + ), + PythonRequirementTarget( + {"requirements": ["repletewateringcan>=7"], "dependencies": [file_addr.spec]}, + Address("", target_name="reqs", generated_name="repletewateringcan"), + ), + PythonRequirementsFileTarget({"source": "requirements.txt"}, file_addr), + }, + ) + + +def test_invalid_req(rule_runner: RuleRunner) -> None: + """Test that we give a nice error message.""" + with engine_error( + contains="Invalid requirement 'Not A Valid Req == 3.7' in requirements.txt at line 3" + ): + assert_python_requirements( + rule_runner, + "python_requirements(name='reqs')", + "\n\nNot A Valid Req == 3.7", + expected_targets=set(), + ) + + # Give a nice error message if it looks like they're using pip VCS-style requirements. + with engine_error(contains="It looks like you're trying to use a pip VCS-style requirement?"): + assert_python_requirements( + rule_runner, + "python_requirements(name='reqs')", + "git+https://github.com/pypa/pip.git#egg=pip", + expected_targets=set(), + ) + + +def test_source_override(rule_runner: RuleRunner) -> None: + file_addr = Address("", target_name="reqs", relative_file_path="subdir/requirements.txt") + assert_python_requirements( + rule_runner, + "python_requirements(name='reqs', source='subdir/requirements.txt')", + "ansicolors>=1.18.0", + requirements_txt_relpath="subdir/requirements.txt", + expected_targets={ + PythonRequirementTarget( + {"requirements": ["ansicolors>=1.18.0"], "dependencies": [file_addr.spec]}, + Address("", target_name="reqs", generated_name="ansicolors"), + ), + PythonRequirementsFileTarget({"source": "subdir/requirements.txt"}, file_addr), + }, + )