From eccdb7a817be4b9a13eef2e1f88dd4485226e538 Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 8 Apr 2021 15:17:14 -0700 Subject: [PATCH 01/11] Implemented base for poetry parsing # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] --- .../backend/python/macros/poetry_project.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/python/pants/backend/python/macros/poetry_project.py diff --git a/src/python/pants/backend/python/macros/poetry_project.py b/src/python/pants/backend/python/macros/poetry_project.py new file mode 100644 index 00000000000..ea2b20f4eb7 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_project.py @@ -0,0 +1,65 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + + +from __future__ import annotations + +from typing import Any + +# from pants.util.ordered_set import FrozenOrderedSet +import toml +from pkg_resources import Requirement + +toml2_str_ex = """[tool.poetry] +name = "poetry_tinker" +version = "0.1.0" +description = "" +authors = ["Liam Wilson "] + +[tool.poetry.dependencies] +python = "^3.8" +poetry = {git = "https://github.com/python-poetry/poetry.git"} +requests = {extras = ["security"], version = "^2.25.1"} + +[tool.poetry.dev-dependencies] +isort = ">=5.5.1,<5.6" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +""" + +toml_str_ex = """[tool.poetry] +name = "poetry-demo" +version = "0.1.0" +description = "" +authors = ["Eric Arellano "] +[tool.poetry.dependencies] +# python = "==3.8" +"pantsbuild.pants" = ">=2.2.0" +[tool.poetry.dev-dependencies] +"foo" = ">1.0.0" +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" +""" + + +def parse_pyproject_toml(toml_contents: str) -> set[Requirement]: + parsed = toml.loads(toml_contents) + + def parse_single_dependency(proj_name: str, attributes: str | dict[str, Any]) -> Requirement: + if isinstance(attributes, str): + return Requirement.parse(f"{proj_name}{attributes}") + + poetry_vals = parsed["tool"]["poetry"] + all_req_raw = {**poetry_vals["dependencies"], **poetry_vals["dev-dependencies"]} + + return {parse_single_dependency(proj, attr) for proj, attr in all_req_raw.items()} + + +def parse_poetry_lock(lock_contents: str) -> set[Requirement]: + pass + + +print(parse_pyproject_toml(toml_str_ex)) From 5dcb9f32f91ac6bdf8b7c7240b50b1fda9cf9ad1 Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 3 Jun 2021 05:14:36 -0700 Subject: [PATCH 02/11] Fully working but bad error handling # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../backend/python/macros/poetry_project.py | 233 +++++++++++++---- .../python/macros/poetry_project_test.py | 235 ++++++++++++++++++ .../python/macros/poetry_requirement.py | 84 +++++++ .../python/macros/poetry_requirement_test.py | 150 +++++++++++ 4 files changed, 651 insertions(+), 51 deletions(-) create mode 100644 src/python/pants/backend/python/macros/poetry_project_test.py create mode 100644 src/python/pants/backend/python/macros/poetry_requirement.py create mode 100644 src/python/pants/backend/python/macros/poetry_requirement_test.py diff --git a/src/python/pants/backend/python/macros/poetry_project.py b/src/python/pants/backend/python/macros/poetry_project.py index ea2b20f4eb7..4dd436fedf7 100644 --- a/src/python/pants/backend/python/macros/poetry_project.py +++ b/src/python/pants/backend/python/macros/poetry_project.py @@ -1,65 +1,196 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). - from __future__ import annotations -from typing import Any +import itertools +from typing import Any, Optional -# from pants.util.ordered_set import FrozenOrderedSet +import packaging.version import toml from pkg_resources import Requirement -toml2_str_ex = """[tool.poetry] -name = "poetry_tinker" -version = "0.1.0" -description = "" -authors = ["Liam Wilson "] - -[tool.poetry.dependencies] -python = "^3.8" -poetry = {git = "https://github.com/python-poetry/poetry.git"} -requests = {extras = ["security"], version = "^2.25.1"} - -[tool.poetry.dev-dependencies] -isort = ">=5.5.1,<5.6" - -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" -""" - -toml_str_ex = """[tool.poetry] -name = "poetry-demo" -version = "0.1.0" -description = "" -authors = ["Eric Arellano "] -[tool.poetry.dependencies] -# python = "==3.8" -"pantsbuild.pants" = ">=2.2.0" -[tool.poetry.dev-dependencies] -"foo" = ">1.0.0" -[build-system] -requires = ["poetry-core>=1.0.0"] -build-backend = "poetry.core.masonry.api" -""" + +def get_max_caret(proj_name: str, version: str) -> str: + major = "0" + minor = "0" + micro = "0" + + try: + parsed_version = packaging.version.Version(version) + if parsed_version.major != 0: + major = str(parsed_version.major + 1) + elif parsed_version.minor != 0: + minor = str(parsed_version.minor + 1) + elif parsed_version.micro != 0: + micro = str(parsed_version.micro + 1) + else: + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 3: + micro = "1" + elif base_len == 2: + minor = "1" + elif base_len == 1: + major = "1" + except packaging.version.InvalidVersion: + print( + f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" + f" will be left as >={version},<{version}" + ) + return version + return f"{major}.{minor}.{micro}" + + +def get_max_tilde(proj_name: str, version: str) -> str: + major = "0" + minor = "0" + micro = "0" + try: + parsed_version = packaging.version.Version(version) + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 2: + minor = str(parsed_version.minor + 1) + major = str(parsed_version.major) + elif base_len == 1: + major = str(parsed_version.major + 1) + + except packaging.version.InvalidVersion: + print( + f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" + f" will be parsed as >={version},<{version}" + ) + return version + return f"{major}.{minor}.{micro}" + + +def handle_str_attr(proj_name: str, attributes: str) -> str: + valid_specifiers = "<>!~=" + pep440_reqs = [] + comma_split_reqs = [i.strip() for i in attributes.split(",")] + for req in comma_split_reqs: + if req[0] == "^": + max_ver = get_max_caret(proj_name, req[1:]) + min_ver = req[1:] + pep440_reqs.append(f">={min_ver},<{max_ver}") + # ~= is an acceptable default operator; however, ~ is not, and IS NOT the same as ~= + elif req[0] == "~" and req[1] != "=": + max_ver = get_max_tilde(proj_name, req[1:]) + min_ver = req[1:] + pep440_reqs.append(f">={min_ver},<{max_ver}") + else: + if req[0] not in valid_specifiers: + pep440_reqs.append(f"=={req}") + else: + pep440_reqs.append(req) + return f"{proj_name} {','.join(pep440_reqs)}" + + +def parse_python_constraint(constr: str | None) -> str: + if constr is None: + return "" + valid_specifiers = "<>!~= " + or_and_split = [[j.strip() for j in i.split(",")] for i in constr.split("||")] + ver_parsed = [[handle_str_attr("", j) for j in i] for i in or_and_split] + + def conv_and(lst: list[str]) -> list: + return list(itertools.chain(*[i.split(",") for i in lst])) + + def prepend(version: str) -> str: + return ( + f"python_version{''.join(i for i in version if i in valid_specifiers)} '" + f"{''.join(i for i in version if i not in valid_specifiers)}'" + ) + + prepend_and_clean = [ + [prepend(".".join(j.split(".")[:2])) for j in conv_and(i)] for i in ver_parsed + ] + return ( + f"{'(' if len(or_and_split) > 1 else ''}" + f"{') or ('.join([' and '.join(i) for i in prepend_and_clean])}" + f"{')' if len(or_and_split) > 1 else ''}" + ) + + +def handle_dict_attr(proj_name: str, attributes: dict[str, str]) -> str: + def produce_match(sep: str, feat: Optional[str]) -> str: + return f"{sep}{feat}" if feat else "" + + git_lookup = attributes.get("git") + if git_lookup is not None: + # Making revision optional and supporting a branch are not + # strictly in agreement with PEP440 (does not comment on + # specifying a branch). However, pip seems to accept it fine + # (see pypa/pip issue 7650) + # so I can't see a reason not to support it here...? + + rev_lookup = produce_match("#", attributes.get("rev")) + branch_lookup = produce_match("@", attributes.get("branch")) + tag_lookup = produce_match("@", attributes.get("tag")) + + return f"{proj_name} @ git+{git_lookup}{tag_lookup}{branch_lookup}{rev_lookup}" + + version_lookup = attributes.get("version") + path_lookup = attributes.get("path") + if path_lookup is not None: + return f"{proj_name} @ file://{path_lookup}" + url_lookup = attributes.get("url") + if url_lookup is not None: + return f"{proj_name} @ {url_lookup}" + if version_lookup is not None: + markers_lookup = produce_match(";", attributes.get("markers")) + python_lookup = parse_python_constraint(attributes.get("python")) + version_parsed = handle_str_attr(proj_name, version_lookup) + return ( + f"{version_parsed}" + f"{markers_lookup}" + f"{' and ' if python_lookup and markers_lookup else (';' if python_lookup else '')}" + f"{python_lookup}" + ) + else: + # This case should never happen if input is formed correctly; this + # ultimately will cause the Requirement.parse call to fail. + # TODO: add appropriate error handling + return "" + + +def parse_single_dependency( + proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]] +) -> tuple[Requirement, ...]: + if isinstance(attributes, str): + return (Requirement.parse(handle_str_attr(proj_name, attributes)),) + elif isinstance(attributes, dict): + return (Requirement.parse(handle_dict_attr(proj_name, attributes)),) + elif isinstance(attributes, list): + return tuple([Requirement.parse(handle_dict_attr(proj_name, attr)) for attr in attributes]) + else: + # Similarly, this is currently added just for mypy checks. Error + # handling needs to be written to address this properly. + return (Requirement.parse(""),) def parse_pyproject_toml(toml_contents: str) -> set[Requirement]: parsed = toml.loads(toml_contents) - - def parse_single_dependency(proj_name: str, attributes: str | dict[str, Any]) -> Requirement: - if isinstance(attributes, str): - return Requirement.parse(f"{proj_name}{attributes}") - poetry_vals = parsed["tool"]["poetry"] - all_req_raw = {**poetry_vals["dependencies"], **poetry_vals["dev-dependencies"]} - - return {parse_single_dependency(proj, attr) for proj, attr in all_req_raw.items()} - - -def parse_poetry_lock(lock_contents: str) -> set[Requirement]: - pass - - -print(parse_pyproject_toml(toml_str_ex)) + try: + dependencies = poetry_vals["dependencies"] + dev_dependencies = poetry_vals["dev-dependencies"] + except KeyError: + print( + "Missing required `poetry.tools.dependencies` and/or `poetry.tools.dev-dependencies`" + "fields...perhaps you specified the wrong file?" + "These can be empty if, for example, you have no dev-dependencies." + ) + return set() + + all_req_raw = {**dependencies, **dev_dependencies} + + # This cannot be a comprehension, as the special case where one dependency + # needs to be treated as two dependencies (e.g. version requirement + # different depending on environment markers) can return two Requirement + # objects, and comprehensions do not permit asterisk unpacking. + all_process_deps = set() + for proj, attr in all_req_raw.items(): + processed_deps = parse_single_dependency(proj, attr) + for dep in processed_deps: + all_process_deps.add(dep) + return all_process_deps diff --git a/src/python/pants/backend/python/macros/poetry_project_test.py b/src/python/pants/backend/python/macros/poetry_project_test.py new file mode 100644 index 00000000000..263a3971697 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_project_test.py @@ -0,0 +1,235 @@ +from pkg_resources import Requirement + +from pants.backend.python.macros.poetry_project import ( + get_max_caret, + get_max_tilde, + handle_dict_attr, + handle_str_attr, + parse_pyproject_toml, + parse_single_dependency, +) + +# TODO: pytest parameterize for caret/tilde edge + + +def test_max_caret_1() -> None: + assert get_max_caret("", "1.2.3") == "2.0.0" + + +def test_max_caret_2() -> None: + assert get_max_caret("", "1.2") == "2.0.0" + + +def test_max_caret_3() -> None: + assert get_max_caret("", "1") == "2.0.0" + + +def test_max_caret_4() -> None: + assert get_max_caret("", "0.2.3") == "0.3.0" + + +def test_max_caret_5() -> None: + assert get_max_caret("", "0.0.3") == "0.0.4" + + +def test_max_caret_6() -> None: + assert get_max_caret("", "0.0") == "0.1.0" + + +def test_max_caret_7() -> None: + assert get_max_caret("", "0") == "1.0.0" + + +def test_max_tilde_1() -> None: + assert get_max_tilde("", "1.2.3") == "1.3.0" + + +def test_max_tilde_2() -> None: + assert get_max_tilde("", "1.2") == "1.3.0" + + +def test_max_tilde_3() -> None: + assert get_max_tilde("", "1") == "2.0.0" + + +def test_max_tilde_4() -> None: + assert get_max_tilde("", "0") == "1.0.0" + + +def test_handle_str_tilde() -> None: + assert handle_str_attr("foo", "~1.2.3") == "foo >=1.2.3,<1.3.0" + + +def test_handle_str_caret() -> None: + assert handle_str_attr("foo", "^1.2.3") == "foo >=1.2.3,<2.0.0" + + +def test_handle_compat_operator() -> None: + assert handle_str_attr("foo", "~=1.2.3") == "foo ~=1.2.3" + + +def test_handle_no_operator() -> None: + assert handle_str_attr("foo", "1.2.3") == "foo ==1.2.3" + + +def test_handle_one_char_operator() -> None: + assert handle_str_attr("foo", ">1.2.3") == "foo >1.2.3" + + +def test_handle_multiple_reqs() -> None: + assert handle_str_attr("foo", "~1.2, !=1.2.10") == "foo >=1.2,<1.3.0,!=1.2.10" + + +def test_handle_git_basic() -> None: + attr = {"git": "https://github.com/requests/requests.git"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git" + ) + + +# TODO: conglomerate git/etc with kwargs (parameterize dict) +def test_handle_git_branch() -> None: + attr = {"git": "https://github.com/requests/requests.git", "branch": "main"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git@main" + ) + + +def test_handle_git_tag() -> None: + attr = {"git": "https://github.com/requests/requests.git", "tag": "v1.1.1"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git@v1.1.1" + ) + + +def test_handle_git_revision() -> None: + attr = {"git": "https://github.com/requests/requests.git", "rev": "1a2b3c4d"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git#1a2b3c4d" + ) + + +def test_handle_path_arg() -> None: + attr = {"path": "../../my_py_proj.whl"} + assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ file://../../my_py_proj.whl" + + +def test_handle_url_arg() -> None: + attr = {"url": "https://my-site.com/mydep.whl"} + assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ https://my-site.com/mydep.whl" + + +def test_version_only() -> None: + attr = {"version": "1.2.3"} + assert handle_dict_attr("foo", attr) == "foo ==1.2.3" + + +def test_py_constraint_single() -> None: + attr = {"version": "1.2.3", "python": "3.6"} + assert handle_dict_attr("foo", attr) == "foo ==1.2.3;python_version == '3.6'" + + +def test_py_constraint_or() -> None: + attr = {"version": "1.2.3", "python": "3.6 || 3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version == '3.6') or (python_version == '3.7')" + ) + + +def test_py_constraint_and() -> None: + attr = {"version": "1.2.3", "python": ">3.6,!=3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;python_version > '3.6' and python_version != '3.7'" + ) + + +def test_py_constraint_and_or() -> None: + attr = {"version": "1.2.3", "python": ">3.6 || 3.5,3.4"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')" + ) + + +def test_py_constraint_tilde_caret_and_or() -> None: + attr = {"version": "1.2.3", "python": "~3.6 || ^3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')" + ) + + +def test_multi_version_const() -> None: + lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] + retval = parse_single_dependency("foo", lst_attr) + actual_reqs = ( + Requirement.parse("foo ==1.2.3; python_version == '3.6'"), + Requirement.parse("foo ==1.2.4; python_version == '3.7'"), + ) + assert retval == actual_reqs + + +def test_extended_form() -> None: + toml_black_str = """ + [tool.poetry.dependencies] + [tool.poetry.dependencies.black] + version = "19.10b0" + python = "3.6" + markers = "platform_python_implementation == 'CPython'" + [tool.poetry.dev-dependencies] + """ + retval = parse_pyproject_toml(toml_black_str) + actual_req = { + Requirement.parse( + 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' + ) + } + assert retval == actual_req + + +def test_parse_multi_reqs() -> None: + toml_str = """[tool.poetry] + name = "poetry_tinker" + version = "0.1.0" + description = "" + authors = ["Liam Wilson "] + + [tool.poetry.dependencies] + python = "^3.8" + junk = {url = "https://github.com/myrepo/junk.whl"} + poetry = {git = "https://github.com/python-poetry/poetry.git", tag = "v1.1.1"} + requests = {extras = ["security"], version = "^2.25.1", python = ">2.7"} + foo = [{version = ">=1.9", python = "^2.7"},{version = "^2.0", python = "3.4 || 3.5"}] + + [tool.poetry.dependencies.black] + version = "19.10b0" + python = "3.6" + markers = "platform_python_implementation == 'CPython'" + + [tool.poetry.dev-dependencies] + isort = ">=5.5.1,<5.6" + + [build-system] + requires = ["poetry-core>=1.0.0"] + build-backend = "poetry.core.masonry.api" + """ + retval = parse_pyproject_toml(toml_str) + actual_reqs = { + Requirement.parse("python<4.0.0,>=3.8"), + Requirement.parse("junk@ https://github.com/myrepo/junk.whl"), + Requirement.parse("poetry@ git+https://github.com/python-poetry/poetry.git@v1.1.1"), + Requirement.parse('requests<3.0.0,>=2.25.1; python_version > "2.7"'), + Requirement.parse('foo>=1.9; python_version >= "2.7" and python_version < "3.0"'), + Requirement.parse('foo<3.0.0,>=2.0; python_version == "3.4" or python_version == "3.5"'), + Requirement.parse( + 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' + ), + Requirement.parse("isort<5.6,>=5.5.1"), + } + assert retval == actual_reqs diff --git a/src/python/pants/backend/python/macros/poetry_requirement.py b/src/python/pants/backend/python/macros/poetry_requirement.py new file mode 100644 index 00000000000..66a08413216 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_requirement.py @@ -0,0 +1,84 @@ +# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). +import os +from pathlib import Path +from typing import Iterable, Mapping, Optional + +from pants.backend.python.macros.poetry_project import parse_pyproject_toml + +# from pants.backend.python.target_types import parse_requirements_file +from pants.base.build_environment import get_buildroot + + +class PoetryRequirements: + """Translates dependencies specified in a pyproject.toml Poetry file to a set of + "python_requirements_library" targets. + + For example, if pyproject.toml contains the following entries under + poetry.tool.dependencies: `foo = "~1"` and `bar = "^2.4"`, + + python_requirement_library( + name="foo", + requirements=["foo>=3.14"], + ) + + python_requirement_library( + name="bar", + requirements=["bar>=2.7"], + ) + + See Poetry documentation for correct specification of pyproject.toml: + https://python-poetry.org/docs/pyproject/ + + You may also use the parameter `module_mapping` to teach Pants what modules each of your + requirements provide. For any requirement unspecified, Pants will default to the name of the + requirement. This setting is important for Pants to know how to convert your import + statements back into your dependencies. For example: + + python_requirements( + module_mapping={ + "ansicolors": ["colors"], + "setuptools": ["pkg_resources"], + } + ) + """ + + def __init__(self, parse_context): + self._parse_context = parse_context + + def __call__( + self, + pyproject_toml_relpath: str = "pyproject.toml", + *, + module_mapping: Optional[Mapping[str, Iterable[str]]] = None, + ) -> None: + """ + :param pyproject_toml_relpath: The relpath from this BUILD file to the requirements file. + Defaults to a `requirements.txt` file sibling to the BUILD file. + :param module_mapping: a mapping of requirement names to a list of the modules they provide. + For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the + requirement name as the default module, e.g. "Django" will default to + `modules=["django"]`. + """ + req_file_tgt = self._parse_context.create_object( + "_python_requirements_file", + name=pyproject_toml_relpath.replace(os.path.sep, "_"), + sources=[pyproject_toml_relpath], + ) + requirements_dep = f":{req_file_tgt.name}" + + req_file = Path(get_buildroot(), self._parse_context.rel_path, pyproject_toml_relpath) + requirements = parse_pyproject_toml(req_file.read_text()) + for parsed_req in requirements: + req_module_mapping = ( + {parsed_req.project_name: module_mapping[parsed_req.project_name]} + if module_mapping and parsed_req.project_name in module_mapping + else None + ) + self._parse_context.create_object( + "python_requirement_library", + name=parsed_req.project_name, + requirements=[parsed_req], + module_mapping=req_module_mapping, + dependencies=[requirements_dep], + ) diff --git a/src/python/pants/backend/python/macros/poetry_requirement_test.py b/src/python/pants/backend/python/macros/poetry_requirement_test.py new file mode 100644 index 00000000000..d7f41901536 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_requirement_test.py @@ -0,0 +1,150 @@ +# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from textwrap import dedent +from typing import Iterable + +import pytest +from pkg_resources import Requirement + +from pants.backend.python.macros.poetry_requirement import PoetryRequirements +from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile +from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs +from pants.engine.addresses import Address +from pants.engine.target import Targets +from pants.testutil.rule_runner import QueryRule, RuleRunner + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[QueryRule(Targets, (Specs,))], + target_types=[PythonRequirementLibrary, PythonRequirementsFile], + context_aware_object_factories={"python_requirements": PoetryRequirements}, + ) + + +def assert_python_requirements( + rule_runner: RuleRunner, + build_file_entry: str, + pyproject_toml: str, + *, + expected_file_dep: PythonRequirementsFile, + expected_targets: Iterable[PythonRequirementLibrary], + pyproject_toml_relpath: str = "pyproject.toml", +) -> None: + rule_runner.add_to_build_file("", f"{build_file_entry}\n") + rule_runner.create_file(pyproject_toml_relpath, pyproject_toml) + targets = rule_runner.request( + Targets, + [Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))], + ) + assert {expected_file_dep, *expected_targets} == set(targets) + + +def test_pyproject_toml(rule_runner: RuleRunner) -> None: + """This tests that we correctly create a new python_requirement_library for each entry in a + pyproject.toml file. + + Note that this just ensures proper targets are created; whether or not all dependencies are + parsed are checked in tests found in poetry_project_test.py. + """ + assert_python_requirements( + rule_runner, + "python_requirements(module_mapping={'ansicolors': ['colors']})", + dedent( + """\ + [tool.poetry.dependencies] + Django = {version = "3.2", python = "3"} + ansicolors = ">=1.18.0" + Un-Normalized-PROJECT = "1.0.0" + [tool.poetry.dev-dependencies] + """ + ), + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("ansicolors>=1.18.0")], + "module_mapping": {"ansicolors": ["colors"]}, + }, + address=Address("", target_name="ansicolors"), + ), + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("Django==3.2 ; python_version == '3'")], + }, + address=Address("", target_name="Django"), + ), + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("Un_Normalized_PROJECT == 1.0.0")], + }, + address=Address("", target_name="Un-Normalized-PROJECT"), + ), + ], + ) + + +def test_relpath_override(rule_runner: RuleRunner) -> None: + assert_python_requirements( + rule_runner, + "python_requirements(pyproject_toml_relpath='subdir/pyproject.toml')", + dedent( + """\ + [tool.poetry.dependencies] + ansicolors = ">=1.18.0" + [tool.poetry.dev-dependencies] + """ + ), + pyproject_toml_relpath="subdir/pyproject.toml", + expected_file_dep=PythonRequirementsFile( + {"sources": ["subdir/pyproject.toml"]}, + address=Address("", target_name="subdir_pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":subdir_pyproject.toml"], + "requirements": [Requirement.parse("ansicolors>=1.18.0")], + }, + address=Address("", target_name="ansicolors"), + ), + ], + ) + + +# TODO: tests currently broken; fix/finish & add appropriate error handling. +def test_bad_version(rule_runner: RuleRunner, capf) -> None: + assert_python_requirements( + rule_runner, + "python_requirements()", + """ + [tool.poetry.dependencies] + foo = "~r62b" + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("foo =r62b")], + }, + address=Address("", target_name="foo"), + ) + ], + ) + + +def test_bad_toml_format() -> None: + pass From bab4c50c0e6f99e92257f49920ebf98e22e9be83 Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 3 Jun 2021 05:19:07 -0700 Subject: [PATCH 03/11] Fixed header for test file # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/macros/poetry_project_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/python/pants/backend/python/macros/poetry_project_test.py b/src/python/pants/backend/python/macros/poetry_project_test.py index 263a3971697..2320b51116e 100644 --- a/src/python/pants/backend/python/macros/poetry_project_test.py +++ b/src/python/pants/backend/python/macros/poetry_project_test.py @@ -1,3 +1,6 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + from pkg_resources import Requirement from pants.backend.python.macros.poetry_project import ( From 2d53a866f97b233fe578fda6c0c02c14a7aef9c0 Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 17 Jun 2021 02:57:24 -0700 Subject: [PATCH 04/11] Mostly error work with some cleanup # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../backend/python/macros/poetry_project.py | 127 +++--- .../python/macros/poetry_project_test.py | 180 +++++--- .../python/macros/poetry_requirement.py | 12 +- .../python/macros/poetry_requirement_test.py | 101 ++++- .../python/macros/poetry_requirements.py | 287 ++++++++++++ .../python/macros/poetry_requirements_test.py | 414 ++++++++++++++++++ 6 files changed, 973 insertions(+), 148 deletions(-) create mode 100644 src/python/pants/backend/python/macros/poetry_requirements.py create mode 100644 src/python/pants/backend/python/macros/poetry_requirements_test.py diff --git a/src/python/pants/backend/python/macros/poetry_project.py b/src/python/pants/backend/python/macros/poetry_project.py index 4dd436fedf7..dbce3b58922 100644 --- a/src/python/pants/backend/python/macros/poetry_project.py +++ b/src/python/pants/backend/python/macros/poetry_project.py @@ -4,12 +4,15 @@ from __future__ import annotations import itertools +import logging from typing import Any, Optional import packaging.version import toml from pkg_resources import Requirement +logger = logging.getLogger(__name__) + def get_max_caret(proj_name: str, version: str) -> str: major = "0" @@ -18,26 +21,28 @@ def get_max_caret(proj_name: str, version: str) -> str: try: parsed_version = packaging.version.Version(version) - if parsed_version.major != 0: - major = str(parsed_version.major + 1) - elif parsed_version.minor != 0: - minor = str(parsed_version.minor + 1) - elif parsed_version.micro != 0: - micro = str(parsed_version.micro + 1) - else: - base_len = len(parsed_version.base_version.split(".")) - if base_len >= 3: - micro = "1" - elif base_len == 2: - minor = "1" - elif base_len == 1: - major = "1" except packaging.version.InvalidVersion: - print( + logger.warning( f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" f" will be left as >={version},<{version}" ) return version + + if parsed_version.major != 0: + major = str(parsed_version.major + 1) + elif parsed_version.minor != 0: + minor = str(parsed_version.minor + 1) + elif parsed_version.micro != 0: + micro = str(parsed_version.micro + 1) + else: + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 3: + micro = "1" + elif base_len == 2: + minor = "1" + elif base_len == 1: + major = "1" + return f"{major}.{minor}.{micro}" @@ -47,23 +52,24 @@ def get_max_tilde(proj_name: str, version: str) -> str: micro = "0" try: parsed_version = packaging.version.Version(version) - base_len = len(parsed_version.base_version.split(".")) - if base_len >= 2: - minor = str(parsed_version.minor + 1) - major = str(parsed_version.major) - elif base_len == 1: - major = str(parsed_version.major + 1) - except packaging.version.InvalidVersion: - print( + logger.warning( f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" f" will be parsed as >={version},<{version}" ) return version + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 2: + minor = str(parsed_version.minor + 1) + major = str(parsed_version.major) + elif base_len == 1: + major = str(parsed_version.major + 1) + return f"{major}.{minor}.{micro}" def handle_str_attr(proj_name: str, attributes: str) -> str: + # kwarg for parse_python_constraint valid_specifiers = "<>!~=" pep440_reqs = [] comma_split_reqs = [i.strip() for i in attributes.split(",")] @@ -117,12 +123,6 @@ def produce_match(sep: str, feat: Optional[str]) -> str: git_lookup = attributes.get("git") if git_lookup is not None: - # Making revision optional and supporting a branch are not - # strictly in agreement with PEP440 (does not comment on - # specifying a branch). However, pip seems to accept it fine - # (see pypa/pip issue 7650) - # so I can't see a reason not to support it here...? - rev_lookup = produce_match("#", attributes.get("rev")) branch_lookup = produce_match("@", attributes.get("branch")) tag_lookup = produce_match("@", attributes.get("tag")) @@ -147,10 +147,13 @@ def produce_match(sep: str, feat: Optional[str]) -> str: f"{python_lookup}" ) else: - # This case should never happen if input is formed correctly; this - # ultimately will cause the Requirement.parse call to fail. - # TODO: add appropriate error handling - return "" + raise AssertionError( + ( + f"{proj_name} is not formatted correctly; at" + " minimum provide either a version, url, path or git location for" + " your dependency. " + ) + ) def parse_single_dependency( @@ -163,34 +166,42 @@ def parse_single_dependency( elif isinstance(attributes, list): return tuple([Requirement.parse(handle_dict_attr(proj_name, attr)) for attr in attributes]) else: - # Similarly, this is currently added just for mypy checks. Error - # handling needs to be written to address this properly. - return (Requirement.parse(""),) + raise AssertionError( + ( + "Error: invalid poetry requirement format. Expected " + " type of requirement attributes to be string," + f"dict, or list, but was of type {type(attributes).__name__}." + ) + ) -def parse_pyproject_toml(toml_contents: str) -> set[Requirement]: +def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement]: parsed = toml.loads(toml_contents) - poetry_vals = parsed["tool"]["poetry"] try: - dependencies = poetry_vals["dependencies"] - dev_dependencies = poetry_vals["dev-dependencies"] + poetry_vals = parsed["tool"]["poetry"] except KeyError: - print( - "Missing required `poetry.tools.dependencies` and/or `poetry.tools.dev-dependencies`" - "fields...perhaps you specified the wrong file?" - "These can be empty if, for example, you have no dev-dependencies." + raise KeyError( + ( + f"No section `tool.poetry` found in {file_path}, which" + "is loaded by Pants from a `poetry_requirements` macro. " + "Did you mean to set up Poetry?" + ) ) - return set() - - all_req_raw = {**dependencies, **dev_dependencies} - - # This cannot be a comprehension, as the special case where one dependency - # needs to be treated as two dependencies (e.g. version requirement - # different depending on environment markers) can return two Requirement - # objects, and comprehensions do not permit asterisk unpacking. - all_process_deps = set() - for proj, attr in all_req_raw.items(): - processed_deps = parse_single_dependency(proj, attr) - for dep in processed_deps: - all_process_deps.add(dep) - return all_process_deps + dependencies = poetry_vals.get("dependencies", {}) + dev_dependencies = poetry_vals.get("dev-dependencies", {}) + if not dependencies and not dev_dependencies: + logger.warning( + ( + "No requirements defined in poetry.tools.dependencies and" + f" poetry.tools.dev-dependencies in {file_path}, which is loaded by Pants" + " from a poetry_requirements macro. Did you mean to populate these" + " with requirements?" + ) + ) + + return set( + itertools.chain.from_iterable( + parse_single_dependency(proj, attr) + for proj, attr in {**dependencies, **dev_dependencies}.items() + ) + ) diff --git a/src/python/pants/backend/python/macros/poetry_project_test.py b/src/python/pants/backend/python/macros/poetry_project_test.py index 2320b51116e..fe688a0e4d7 100644 --- a/src/python/pants/backend/python/macros/poetry_project_test.py +++ b/src/python/pants/backend/python/macros/poetry_project_test.py @@ -1,9 +1,10 @@ # Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import pytest from pkg_resources import Requirement -from pants.backend.python.macros.poetry_project import ( +from pants.backend.python.macros.poetry_requirements import ( get_max_caret, get_max_tilde, handle_dict_attr, @@ -12,9 +13,24 @@ parse_single_dependency, ) -# TODO: pytest parameterize for caret/tilde edge + +@pytest.mark.parametrize( + "test, exp", + [ + ("1.2.3", "2.0.0"), + ("1.2", "2.0.0"), + ("1", "2.0.0"), + ("0.2.3", "0.3.0"), + ("0.0.3", "0.0.4"), + ("0.0", "0.1.0"), + ("0", "1.0.0"), + ], +) +def test_caret(test, exp) -> None: + assert get_max_caret("", test) == exp +""" def test_max_caret_1() -> None: assert get_max_caret("", "1.2.3") == "2.0.0" @@ -42,7 +58,17 @@ def test_max_caret_6() -> None: def test_max_caret_7() -> None: assert get_max_caret("", "0") == "1.0.0" +""" + + +@pytest.mark.parametrize( + "test, exp", [("1.2.3", "1.3.0"), ("1.2", "1.3.0"), ("1", "2.0.0"), ("0", "1.0.0")] +) +def test_max_tilde(test, exp) -> None: + assert get_max_tilde("", test) == exp + +""" def test_max_tilde_1() -> None: assert get_max_tilde("", "1.2.3") == "1.3.0" @@ -57,8 +83,25 @@ def test_max_tilde_3() -> None: def test_max_tilde_4() -> None: assert get_max_tilde("", "0") == "1.0.0" +""" + + +@pytest.mark.parametrize( + "test, exp", + [ + ("~1.2.3", ">=1.2.3,<1.3.0"), + ("^1.2.3", ">=1.2.3,<2.0.0"), + ("~=1.2.3", "~=1.2.3"), + ("1.2.3", "==1.2.3"), + (">1.2.3", ">1.2.3"), + ("~1.2, !=1.2.10", ">=1.2,<1.3.0,!=1.2.10"), + ], +) +def test_handle_str(test, exp) -> None: + assert handle_str_attr("foo", test) == f"foo {exp}" +""" def test_handle_str_tilde() -> None: assert handle_str_attr("foo", "~1.2.3") == "foo >=1.2.3,<1.3.0" @@ -82,38 +125,37 @@ def test_handle_one_char_operator() -> None: def test_handle_multiple_reqs() -> None: assert handle_str_attr("foo", "~1.2, !=1.2.10") == "foo >=1.2,<1.3.0,!=1.2.10" +""" -def test_handle_git_basic() -> None: - attr = {"git": "https://github.com/requests/requests.git"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git" - ) - - -# TODO: conglomerate git/etc with kwargs (parameterize dict) -def test_handle_git_branch() -> None: - attr = {"git": "https://github.com/requests/requests.git", "branch": "main"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git@main" - ) +def test_handle_git() -> None: + def test_handle_git_basic() -> None: + attr = {"git": "https://github.com/requests/requests.git"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git" + ) -def test_handle_git_tag() -> None: - attr = {"git": "https://github.com/requests/requests.git", "tag": "v1.1.1"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git@v1.1.1" - ) + def test_handle_git_branch() -> None: + attr = {"git": "https://github.com/requests/requests.git", "branch": "main"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git@main" + ) + def test_handle_git_tag() -> None: + attr = {"git": "https://github.com/requests/requests.git", "tag": "v1.1.1"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git@v1.1.1" + ) -def test_handle_git_revision() -> None: - attr = {"git": "https://github.com/requests/requests.git", "rev": "1a2b3c4d"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git#1a2b3c4d" - ) + def test_handle_git_revision() -> None: + attr = {"git": "https://github.com/requests/requests.git", "rev": "1a2b3c4d"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git#1a2b3c4d" + ) def test_handle_path_arg() -> None: @@ -131,51 +173,47 @@ def test_version_only() -> None: assert handle_dict_attr("foo", attr) == "foo ==1.2.3" -def test_py_constraint_single() -> None: - attr = {"version": "1.2.3", "python": "3.6"} - assert handle_dict_attr("foo", attr) == "foo ==1.2.3;python_version == '3.6'" - - -def test_py_constraint_or() -> None: - attr = {"version": "1.2.3", "python": "3.6 || 3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version == '3.6') or (python_version == '3.7')" - ) - - -def test_py_constraint_and() -> None: - attr = {"version": "1.2.3", "python": ">3.6,!=3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;python_version > '3.6' and python_version != '3.7'" - ) - +def test_py_constraints() -> None: + def test_py_constraint_single() -> None: + attr = {"version": "1.2.3", "python": "3.6"} + assert handle_dict_attr("foo", attr) == "foo ==1.2.3;python_version == '3.6'" -def test_py_constraint_and_or() -> None: - attr = {"version": "1.2.3", "python": ">3.6 || 3.5,3.4"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')" - ) + def test_py_constraint_or() -> None: + attr = {"version": "1.2.3", "python": "3.6 || 3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version == '3.6') or (python_version == '3.7')" + ) + def test_py_constraint_and() -> None: + attr = {"version": "1.2.3", "python": ">3.6,!=3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;python_version > '3.6' and python_version != '3.7'" + ) -def test_py_constraint_tilde_caret_and_or() -> None: - attr = {"version": "1.2.3", "python": "~3.6 || ^3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')" - ) + def test_py_constraint_and_or() -> None: + attr = {"version": "1.2.3", "python": ">3.6 || 3.5,3.4"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')" + ) + def test_py_constraint_tilde_caret_and_or() -> None: + attr = {"version": "1.2.3", "python": "~3.6 || ^3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')" + ) -def test_multi_version_const() -> None: - lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] - retval = parse_single_dependency("foo", lst_attr) - actual_reqs = ( - Requirement.parse("foo ==1.2.3; python_version == '3.6'"), - Requirement.parse("foo ==1.2.4; python_version == '3.7'"), - ) - assert retval == actual_reqs + def test_multi_version_const() -> None: + lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] + retval = parse_single_dependency("foo", lst_attr) + actual_reqs = ( + Requirement.parse("foo ==1.2.3; python_version == '3.6'"), + Requirement.parse("foo ==1.2.4; python_version == '3.7'"), + ) + assert retval == actual_reqs def test_extended_form() -> None: @@ -187,7 +225,7 @@ def test_extended_form() -> None: markers = "platform_python_implementation == 'CPython'" [tool.poetry.dev-dependencies] """ - retval = parse_pyproject_toml(toml_black_str) + retval = parse_pyproject_toml(toml_black_str, "/path/to/file") actual_req = { Requirement.parse( 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' @@ -222,7 +260,7 @@ def test_parse_multi_reqs() -> None: requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" """ - retval = parse_pyproject_toml(toml_str) + retval = parse_pyproject_toml(toml_str, "/path/to/file") actual_reqs = { Requirement.parse("python<4.0.0,>=3.8"), Requirement.parse("junk@ https://github.com/myrepo/junk.whl"), diff --git a/src/python/pants/backend/python/macros/poetry_requirement.py b/src/python/pants/backend/python/macros/poetry_requirement.py index 66a08413216..8262beade97 100644 --- a/src/python/pants/backend/python/macros/poetry_requirement.py +++ b/src/python/pants/backend/python/macros/poetry_requirement.py @@ -1,4 +1,4 @@ -# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). import os from pathlib import Path @@ -15,16 +15,16 @@ class PoetryRequirements: "python_requirements_library" targets. For example, if pyproject.toml contains the following entries under - poetry.tool.dependencies: `foo = "~1"` and `bar = "^2.4"`, + poetry.tool.dependencies: `foo = ">1"` and `bar = ">2.4"`, python_requirement_library( name="foo", - requirements=["foo>=3.14"], + requirements=["foo>1"], ) python_requirement_library( name="bar", - requirements=["bar>=2.7"], + requirements=["bar>2.4"], ) See Poetry documentation for correct specification of pyproject.toml: @@ -68,7 +68,9 @@ def __call__( requirements_dep = f":{req_file_tgt.name}" req_file = Path(get_buildroot(), self._parse_context.rel_path, pyproject_toml_relpath) - requirements = parse_pyproject_toml(req_file.read_text()) + requirements = parse_pyproject_toml( + req_file.read_text(), str(req_file.relative_to(get_buildroot())) + ) for parsed_req in requirements: req_module_mapping = ( {parsed_req.project_name: module_mapping[parsed_req.project_name]} diff --git a/src/python/pants/backend/python/macros/poetry_requirement_test.py b/src/python/pants/backend/python/macros/poetry_requirement_test.py index d7f41901536..a878e7ff760 100644 --- a/src/python/pants/backend/python/macros/poetry_requirement_test.py +++ b/src/python/pants/backend/python/macros/poetry_requirement_test.py @@ -1,30 +1,35 @@ -# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import logging from textwrap import dedent from typing import Iterable import pytest +from _pytest.logging import LogCaptureFixture from pkg_resources import Requirement -from pants.backend.python.macros.poetry_requirement import PoetryRequirements +from pants.backend.python.macros.poetry_requirements import PoetryRequirements from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError from pants.engine.target import Targets from pants.testutil.rule_runner import QueryRule, RuleRunner +logger = logging.getLogger(__name__) + @pytest.fixture def rule_runner() -> RuleRunner: return RuleRunner( rules=[QueryRule(Targets, (Specs,))], target_types=[PythonRequirementLibrary, PythonRequirementsFile], - context_aware_object_factories={"python_requirements": PoetryRequirements}, + context_aware_object_factories={"poetry_requirements": PoetryRequirements}, ) -def assert_python_requirements( +def assert_poetry_requirements( rule_runner: RuleRunner, build_file_entry: str, pyproject_toml: str, @@ -49,9 +54,9 @@ def test_pyproject_toml(rule_runner: RuleRunner) -> None: Note that this just ensures proper targets are created; whether or not all dependencies are parsed are checked in tests found in poetry_project_test.py. """ - assert_python_requirements( + assert_poetry_requirements( rule_runner, - "python_requirements(module_mapping={'ansicolors': ['colors']})", + "poetry_requirements(module_mapping={'ansicolors': ['colors']})", dedent( """\ [tool.poetry.dependencies] @@ -93,9 +98,9 @@ def test_pyproject_toml(rule_runner: RuleRunner) -> None: def test_relpath_override(rule_runner: RuleRunner) -> None: - assert_python_requirements( + assert_poetry_requirements( rule_runner, - "python_requirements(pyproject_toml_relpath='subdir/pyproject.toml')", + "poetry_requirements(pyproject_toml_relpath='subdir/pyproject.toml')", dedent( """\ [tool.poetry.dependencies] @@ -120,11 +125,10 @@ def test_relpath_override(rule_runner: RuleRunner) -> None: ) -# TODO: tests currently broken; fix/finish & add appropriate error handling. -def test_bad_version(rule_runner: RuleRunner, capf) -> None: - assert_python_requirements( +def test_non_pep440_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: + assert_poetry_requirements( rule_runner, - "python_requirements()", + "poetry_requirements()", """ [tool.poetry.dependencies] foo = "~r62b" @@ -144,7 +148,76 @@ def test_bad_version(rule_runner: RuleRunner, capf) -> None: ) ], ) + assert "PEP440" in caplog.text + + +def test_no_req_defined_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "No requirements defined" in caplog.text -def test_bad_toml_format() -> None: - pass +def test_bad_dict_format(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = {bad_req = "test"} + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "not formatted correctly; at" in str(exc.value) + + +def test_bad_req_type(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = 4 + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "was of type int" in str(exc.value) + + +def test_no_tool_poetry(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + foo = 4 + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "`tool.poetry` found in pyproject.toml" in str(exc.value) diff --git a/src/python/pants/backend/python/macros/poetry_requirements.py b/src/python/pants/backend/python/macros/poetry_requirements.py new file mode 100644 index 00000000000..46699a9b933 --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_requirements.py @@ -0,0 +1,287 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import itertools +import logging +import os +from pathlib import Path +from typing import Any, Iterable, Mapping, Optional + +import toml +from packaging.version import InvalidVersion, Version +from pkg_resources import Requirement + +from pants.base.build_environment import get_buildroot + +logger = logging.getLogger(__name__) + + +def get_max_caret(proj_name: str, version: str) -> str: + major = "0" + minor = "0" + micro = "0" + + try: + parsed_version = Version(version) + except InvalidVersion: + logger.warning( + f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" + f" will be left as >={version},<{version}" + ) + return version + + if parsed_version.major != 0: + major = str(parsed_version.major + 1) + elif parsed_version.minor != 0: + minor = str(parsed_version.minor + 1) + elif parsed_version.micro != 0: + micro = str(parsed_version.micro + 1) + else: + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 3: + micro = "1" + elif base_len == 2: + minor = "1" + elif base_len == 1: + major = "1" + + return f"{major}.{minor}.{micro}" + + +def get_max_tilde(proj_name: str, version: str) -> str: + major = "0" + minor = "0" + micro = "0" + try: + parsed_version = Version(version) + except InvalidVersion: + logger.warning( + f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" + f" will be parsed as >={version},<{version}" + ) + return version + base_len = len(parsed_version.base_version.split(".")) + if base_len >= 2: + minor = str(parsed_version.minor + 1) + major = str(parsed_version.major) + elif base_len == 1: + major = str(parsed_version.major + 1) + + return f"{major}.{minor}.{micro}" + + +def handle_str_attr(proj_name: str, attributes: str) -> str: + # kwarg for parse_python_constraint + valid_specifiers = "<>!~=" + pep440_reqs = [] + comma_split_reqs = [i.strip() for i in attributes.split(",")] + for req in comma_split_reqs: + if req[0] == "^": + max_ver = get_max_caret(proj_name, req[1:]) + min_ver = req[1:] + pep440_reqs.append(f">={min_ver},<{max_ver}") + # ~= is an acceptable default operator; however, ~ is not, and IS NOT the same as ~= + elif req[0] == "~" and req[1] != "=": + max_ver = get_max_tilde(proj_name, req[1:]) + min_ver = req[1:] + pep440_reqs.append(f">={min_ver},<{max_ver}") + else: + if req[0] not in valid_specifiers: + pep440_reqs.append(f"=={req}") + else: + pep440_reqs.append(req) + return f"{proj_name} {','.join(pep440_reqs)}" + + +def parse_python_constraint(constr: str | None) -> str: + if constr is None: + return "" + valid_specifiers = "<>!~= " + or_and_split = [[j.strip() for j in i.split(",")] for i in constr.split("||")] + ver_parsed = [[handle_str_attr("", j) for j in i] for i in or_and_split] + + def conv_and(lst: list[str]) -> list: + return list(itertools.chain(*[i.split(",") for i in lst])) + + def prepend(version: str) -> str: + return ( + f"python_version{''.join(i for i in version if i in valid_specifiers)} '" + f"{''.join(i for i in version if i not in valid_specifiers)}'" + ) + + prepend_and_clean = [ + [prepend(".".join(j.split(".")[:2])) for j in conv_and(i)] for i in ver_parsed + ] + return ( + f"{'(' if len(or_and_split) > 1 else ''}" + f"{') or ('.join([' and '.join(i) for i in prepend_and_clean])}" + f"{')' if len(or_and_split) > 1 else ''}" + ) + + +def handle_dict_attr(proj_name: str, attributes: dict[str, str]) -> str: + def produce_match(sep: str, feat: Optional[str]) -> str: + return f"{sep}{feat}" if feat else "" + + git_lookup = attributes.get("git") + if git_lookup is not None: + rev_lookup = produce_match("#", attributes.get("rev")) + branch_lookup = produce_match("@", attributes.get("branch")) + tag_lookup = produce_match("@", attributes.get("tag")) + + return f"{proj_name} @ git+{git_lookup}{tag_lookup}{branch_lookup}{rev_lookup}" + + version_lookup = attributes.get("version") + path_lookup = attributes.get("path") + if path_lookup is not None: + return f"{proj_name} @ file://{path_lookup}" + url_lookup = attributes.get("url") + if url_lookup is not None: + return f"{proj_name} @ {url_lookup}" + if version_lookup is not None: + markers_lookup = produce_match(";", attributes.get("markers")) + python_lookup = parse_python_constraint(attributes.get("python")) + version_parsed = handle_str_attr(proj_name, version_lookup) + return ( + f"{version_parsed}" + f"{markers_lookup}" + f"{' and ' if python_lookup and markers_lookup else (';' if python_lookup else '')}" + f"{python_lookup}" + ) + else: + raise AssertionError( + ( + f"{proj_name} is not formatted correctly; at" + " minimum provide either a version, url, path or git location for" + " your dependency. " + ) + ) + + +def parse_single_dependency( + proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]] +) -> tuple[Requirement, ...]: + if isinstance(attributes, str): + return (Requirement.parse(handle_str_attr(proj_name, attributes)),) + elif isinstance(attributes, dict): + return (Requirement.parse(handle_dict_attr(proj_name, attributes)),) + elif isinstance(attributes, list): + return tuple([Requirement.parse(handle_dict_attr(proj_name, attr)) for attr in attributes]) + else: + raise AssertionError( + ( + "Error: invalid poetry requirement format. Expected " + " type of requirement attributes to be string," + f"dict, or list, but was of type {type(attributes).__name__}." + ) + ) + + +def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement]: + parsed = toml.loads(toml_contents) + try: + poetry_vals = parsed["tool"]["poetry"] + except KeyError: + raise KeyError( + ( + f"No section `tool.poetry` found in {file_path}, which" + "is loaded by Pants from a `poetry_requirements` macro. " + "Did you mean to set up Poetry?" + ) + ) + dependencies = poetry_vals.get("dependencies", {}) + dev_dependencies = poetry_vals.get("dev-dependencies", {}) + if not dependencies and not dev_dependencies: + logger.warning( + ( + "No requirements defined in poetry.tools.dependencies and" + f" poetry.tools.dev-dependencies in {file_path}, which is loaded by Pants" + " from a poetry_requirements macro. Did you mean to populate these" + " with requirements?" + ) + ) + + return set( + itertools.chain.from_iterable( + parse_single_dependency(proj, attr) + for proj, attr in {**dependencies, **dev_dependencies}.items() + ) + ) + + +class PoetryRequirements: + """Translates dependencies specified in a pyproject.toml Poetry file to a set of + "python_requirements_library" targets. + + For example, if pyproject.toml contains the following entries under + poetry.tool.dependencies: `foo = ">1"` and `bar = ">2.4"`, + + python_requirement_library( + name="foo", + requirements=["foo>1"], + ) + + python_requirement_library( + name="bar", + requirements=["bar>2.4"], + ) + + See Poetry documentation for correct specification of pyproject.toml: + https://python-poetry.org/docs/pyproject/ + + You may also use the parameter `module_mapping` to teach Pants what modules each of your + requirements provide. For any requirement unspecified, Pants will default to the name of the + requirement. This setting is important for Pants to know how to convert your import + statements back into your dependencies. For example: + + python_requirements( + module_mapping={ + "ansicolors": ["colors"], + "setuptools": ["pkg_resources"], + } + ) + """ + + def __init__(self, parse_context): + self._parse_context = parse_context + + def __call__( + self, + pyproject_toml_relpath: str = "pyproject.toml", + *, + module_mapping: Optional[Mapping[str, Iterable[str]]] = None, + ) -> None: + """ + :param pyproject_toml_relpath: The relpath from this BUILD file to the requirements file. + Defaults to a `requirements.txt` file sibling to the BUILD file. + :param module_mapping: a mapping of requirement names to a list of the modules they provide. + For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the + requirement name as the default module, e.g. "Django" will default to + `modules=["django"]`. + """ + req_file_tgt = self._parse_context.create_object( + "_python_requirements_file", + name=pyproject_toml_relpath.replace(os.path.sep, "_"), + sources=[pyproject_toml_relpath], + ) + requirements_dep = f":{req_file_tgt.name}" + + req_file = Path(get_buildroot(), self._parse_context.rel_path, pyproject_toml_relpath) + requirements = parse_pyproject_toml( + req_file.read_text(), str(req_file.relative_to(get_buildroot())) + ) + for parsed_req in requirements: + req_module_mapping = ( + {parsed_req.project_name: module_mapping[parsed_req.project_name]} + if module_mapping and parsed_req.project_name in module_mapping + else None + ) + self._parse_context.create_object( + "python_requirement_library", + name=parsed_req.project_name, + requirements=[parsed_req], + module_mapping=req_module_mapping, + dependencies=[requirements_dep], + ) diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py new file mode 100644 index 00000000000..126f45334cb --- /dev/null +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -0,0 +1,414 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from textwrap import dedent +from typing import Iterable + +import pytest +from _pytest.logging import LogCaptureFixture +from pkg_resources import Requirement + +from pants.backend.python.macros.poetry_requirements import ( + PoetryRequirements, + get_max_caret, + get_max_tilde, + handle_dict_attr, + handle_str_attr, + parse_pyproject_toml, + parse_single_dependency, +) +from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile +from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs +from pants.engine.addresses import Address +from pants.engine.internals.scheduler import ExecutionError +from pants.engine.target import Targets +from pants.testutil.rule_runner import QueryRule, RuleRunner + + +@pytest.mark.parametrize( + "test, exp", + [ + ("1.2.3", "2.0.0"), + ("1.2", "2.0.0"), + ("1", "2.0.0"), + ("0.2.3", "0.3.0"), + ("0.0.3", "0.0.4"), + ("0.0", "0.1.0"), + ("0", "1.0.0"), + ], +) +def test_caret(test, exp) -> None: + assert get_max_caret("", test) == exp + + +@pytest.mark.parametrize( + "test, exp", [("1.2.3", "1.3.0"), ("1.2", "1.3.0"), ("1", "2.0.0"), ("0", "1.0.0")] +) +def test_max_tilde(test, exp) -> None: + assert get_max_tilde("", test) == exp + + +@pytest.mark.parametrize( + "test, exp", + [ + ("~1.2.3", ">=1.2.3,<1.3.0"), + ("^1.2.3", ">=1.2.3,<2.0.0"), + ("~=1.2.3", "~=1.2.3"), + ("1.2.3", "==1.2.3"), + (">1.2.3", ">1.2.3"), + ("~1.2, !=1.2.10", ">=1.2,<1.3.0,!=1.2.10"), + ], +) +def test_handle_str(test, exp) -> None: + assert handle_str_attr("foo", test) == f"foo {exp}" + + +def test_handle_git() -> None: + def test_handle_git_basic() -> None: + attr = {"git": "https://github.com/requests/requests.git"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git" + ) + + def test_handle_git_branch() -> None: + attr = {"git": "https://github.com/requests/requests.git", "branch": "main"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git@main" + ) + + def test_handle_git_tag() -> None: + attr = {"git": "https://github.com/requests/requests.git", "tag": "v1.1.1"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git@v1.1.1" + ) + + def test_handle_git_revision() -> None: + attr = {"git": "https://github.com/requests/requests.git", "rev": "1a2b3c4d"} + assert ( + handle_dict_attr("requests", attr) + == "requests @ git+https://github.com/requests/requests.git#1a2b3c4d" + ) + + +def test_handle_path_arg() -> None: + attr = {"path": "../../my_py_proj.whl"} + assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ file://../../my_py_proj.whl" + + +def test_handle_url_arg() -> None: + attr = {"url": "https://my-site.com/mydep.whl"} + assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ https://my-site.com/mydep.whl" + + +def test_version_only() -> None: + attr = {"version": "1.2.3"} + assert handle_dict_attr("foo", attr) == "foo ==1.2.3" + + +def test_py_constraints() -> None: + def test_py_constraint_single() -> None: + attr = {"version": "1.2.3", "python": "3.6"} + assert handle_dict_attr("foo", attr) == "foo ==1.2.3;python_version == '3.6'" + + def test_py_constraint_or() -> None: + attr = {"version": "1.2.3", "python": "3.6 || 3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version == '3.6') or (python_version == '3.7')" + ) + + def test_py_constraint_and() -> None: + attr = {"version": "1.2.3", "python": ">3.6,!=3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;python_version > '3.6' and python_version != '3.7'" + ) + + def test_py_constraint_and_or() -> None: + attr = {"version": "1.2.3", "python": ">3.6 || 3.5,3.4"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')" + ) + + def test_py_constraint_tilde_caret_and_or() -> None: + attr = {"version": "1.2.3", "python": "~3.6 || ^3.7"} + assert ( + handle_dict_attr("foo", attr) + == "foo ==1.2.3;(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')" + ) + + def test_multi_version_const() -> None: + lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] + retval = parse_single_dependency("foo", lst_attr) + actual_reqs = ( + Requirement.parse("foo ==1.2.3; python_version == '3.6'"), + Requirement.parse("foo ==1.2.4; python_version == '3.7'"), + ) + assert retval == actual_reqs + + +def test_extended_form() -> None: + toml_black_str = """ + [tool.poetry.dependencies] + [tool.poetry.dependencies.black] + version = "19.10b0" + python = "3.6" + markers = "platform_python_implementation == 'CPython'" + [tool.poetry.dev-dependencies] + """ + retval = parse_pyproject_toml(toml_black_str, "/path/to/file") + actual_req = { + Requirement.parse( + 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' + ) + } + assert retval == actual_req + + +def test_parse_multi_reqs() -> None: + toml_str = """[tool.poetry] + name = "poetry_tinker" + version = "0.1.0" + description = "" + authors = ["Liam Wilson "] + + [tool.poetry.dependencies] + python = "^3.8" + junk = {url = "https://github.com/myrepo/junk.whl"} + poetry = {git = "https://github.com/python-poetry/poetry.git", tag = "v1.1.1"} + requests = {extras = ["security"], version = "^2.25.1", python = ">2.7"} + foo = [{version = ">=1.9", python = "^2.7"},{version = "^2.0", python = "3.4 || 3.5"}] + + [tool.poetry.dependencies.black] + version = "19.10b0" + python = "3.6" + markers = "platform_python_implementation == 'CPython'" + + [tool.poetry.dev-dependencies] + isort = ">=5.5.1,<5.6" + + [build-system] + requires = ["poetry-core>=1.0.0"] + build-backend = "poetry.core.masonry.api" + """ + retval = parse_pyproject_toml(toml_str, "/path/to/file") + actual_reqs = { + Requirement.parse("python<4.0.0,>=3.8"), + Requirement.parse("junk@ https://github.com/myrepo/junk.whl"), + Requirement.parse("poetry@ git+https://github.com/python-poetry/poetry.git@v1.1.1"), + Requirement.parse('requests<3.0.0,>=2.25.1; python_version > "2.7"'), + Requirement.parse('foo>=1.9; python_version >= "2.7" and python_version < "3.0"'), + Requirement.parse('foo<3.0.0,>=2.0; python_version == "3.4" or python_version == "3.5"'), + Requirement.parse( + 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' + ), + Requirement.parse("isort<5.6,>=5.5.1"), + } + assert retval == actual_reqs + + +@pytest.fixture +def rule_runner() -> RuleRunner: + return RuleRunner( + rules=[QueryRule(Targets, (Specs,))], + target_types=[PythonRequirementLibrary, PythonRequirementsFile], + context_aware_object_factories={"poetry_requirements": PoetryRequirements}, + ) + + +def assert_poetry_requirements( + rule_runner: RuleRunner, + build_file_entry: str, + pyproject_toml: str, + *, + expected_file_dep: PythonRequirementsFile, + expected_targets: Iterable[PythonRequirementLibrary], + pyproject_toml_relpath: str = "pyproject.toml", +) -> None: + rule_runner.add_to_build_file("", f"{build_file_entry}\n") + rule_runner.create_file(pyproject_toml_relpath, pyproject_toml) + targets = rule_runner.request( + Targets, + [Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))], + ) + assert {expected_file_dep, *expected_targets} == set(targets) + + +def test_pyproject_toml(rule_runner: RuleRunner) -> None: + """This tests that we correctly create a new python_requirement_library for each entry in a + pyproject.toml file. + + Note that this just ensures proper targets are created; see prior tests for specific parsing + edge cases. + """ + assert_poetry_requirements( + rule_runner, + "poetry_requirements(module_mapping={'ansicolors': ['colors']})", + dedent( + """\ + [tool.poetry.dependencies] + Django = {version = "3.2", python = "3"} + ansicolors = ">=1.18.0" + Un-Normalized-PROJECT = "1.0.0" + [tool.poetry.dev-dependencies] + """ + ), + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("ansicolors>=1.18.0")], + "module_mapping": {"ansicolors": ["colors"]}, + }, + address=Address("", target_name="ansicolors"), + ), + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("Django==3.2 ; python_version == '3'")], + }, + address=Address("", target_name="Django"), + ), + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("Un_Normalized_PROJECT == 1.0.0")], + }, + address=Address("", target_name="Un-Normalized-PROJECT"), + ), + ], + ) + + +def test_relpath_override(rule_runner: RuleRunner) -> None: + assert_poetry_requirements( + rule_runner, + "poetry_requirements(pyproject_toml_relpath='subdir/pyproject.toml')", + dedent( + """\ + [tool.poetry.dependencies] + ansicolors = ">=1.18.0" + [tool.poetry.dev-dependencies] + """ + ), + pyproject_toml_relpath="subdir/pyproject.toml", + expected_file_dep=PythonRequirementsFile( + {"sources": ["subdir/pyproject.toml"]}, + address=Address("", target_name="subdir_pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":subdir_pyproject.toml"], + "requirements": [Requirement.parse("ansicolors>=1.18.0")], + }, + address=Address("", target_name="ansicolors"), + ), + ], + ) + + +def test_non_pep440_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = "~r62b" + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[ + PythonRequirementLibrary( + { + "dependencies": [":pyproject.toml"], + "requirements": [Requirement.parse("foo =r62b")], + }, + address=Address("", target_name="foo"), + ) + ], + ) + assert "PEP440" in caplog.text + + +def test_no_req_defined_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "No requirements defined" in caplog.text + + +def test_bad_dict_format(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = {bad_req = "test"} + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "not formatted correctly; at" in str(exc.value) + + +def test_bad_req_type(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = 4 + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "was of type int" in str(exc.value) + + +def test_no_tool_poetry(rule_runner: RuleRunner) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + foo = 4 + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert "`tool.poetry` found in pyproject.toml" in str(exc.value) From f610c4e36ef4b6b968ef13cfa37933979be75b66 Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 17 Jun 2021 02:58:44 -0700 Subject: [PATCH 05/11] Delete the separated files (leave combined poetry_requirements.py) # Rust tests and lints will be skipped. Delete if not intended. [ci skip-rust] # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../backend/python/macros/poetry_project.py | 207 ------------- .../python/macros/poetry_project_test.py | 276 ------------------ .../python/macros/poetry_requirement.py | 86 ------ .../python/macros/poetry_requirement_test.py | 223 -------------- 4 files changed, 792 deletions(-) delete mode 100644 src/python/pants/backend/python/macros/poetry_project.py delete mode 100644 src/python/pants/backend/python/macros/poetry_project_test.py delete mode 100644 src/python/pants/backend/python/macros/poetry_requirement.py delete mode 100644 src/python/pants/backend/python/macros/poetry_requirement_test.py diff --git a/src/python/pants/backend/python/macros/poetry_project.py b/src/python/pants/backend/python/macros/poetry_project.py deleted file mode 100644 index dbce3b58922..00000000000 --- a/src/python/pants/backend/python/macros/poetry_project.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -from __future__ import annotations - -import itertools -import logging -from typing import Any, Optional - -import packaging.version -import toml -from pkg_resources import Requirement - -logger = logging.getLogger(__name__) - - -def get_max_caret(proj_name: str, version: str) -> str: - major = "0" - minor = "0" - micro = "0" - - try: - parsed_version = packaging.version.Version(version) - except packaging.version.InvalidVersion: - logger.warning( - f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" - f" will be left as >={version},<{version}" - ) - return version - - if parsed_version.major != 0: - major = str(parsed_version.major + 1) - elif parsed_version.minor != 0: - minor = str(parsed_version.minor + 1) - elif parsed_version.micro != 0: - micro = str(parsed_version.micro + 1) - else: - base_len = len(parsed_version.base_version.split(".")) - if base_len >= 3: - micro = "1" - elif base_len == 2: - minor = "1" - elif base_len == 1: - major = "1" - - return f"{major}.{minor}.{micro}" - - -def get_max_tilde(proj_name: str, version: str) -> str: - major = "0" - minor = "0" - micro = "0" - try: - parsed_version = packaging.version.Version(version) - except packaging.version.InvalidVersion: - logger.warning( - f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" - f" will be parsed as >={version},<{version}" - ) - return version - base_len = len(parsed_version.base_version.split(".")) - if base_len >= 2: - minor = str(parsed_version.minor + 1) - major = str(parsed_version.major) - elif base_len == 1: - major = str(parsed_version.major + 1) - - return f"{major}.{minor}.{micro}" - - -def handle_str_attr(proj_name: str, attributes: str) -> str: - # kwarg for parse_python_constraint - valid_specifiers = "<>!~=" - pep440_reqs = [] - comma_split_reqs = [i.strip() for i in attributes.split(",")] - for req in comma_split_reqs: - if req[0] == "^": - max_ver = get_max_caret(proj_name, req[1:]) - min_ver = req[1:] - pep440_reqs.append(f">={min_ver},<{max_ver}") - # ~= is an acceptable default operator; however, ~ is not, and IS NOT the same as ~= - elif req[0] == "~" and req[1] != "=": - max_ver = get_max_tilde(proj_name, req[1:]) - min_ver = req[1:] - pep440_reqs.append(f">={min_ver},<{max_ver}") - else: - if req[0] not in valid_specifiers: - pep440_reqs.append(f"=={req}") - else: - pep440_reqs.append(req) - return f"{proj_name} {','.join(pep440_reqs)}" - - -def parse_python_constraint(constr: str | None) -> str: - if constr is None: - return "" - valid_specifiers = "<>!~= " - or_and_split = [[j.strip() for j in i.split(",")] for i in constr.split("||")] - ver_parsed = [[handle_str_attr("", j) for j in i] for i in or_and_split] - - def conv_and(lst: list[str]) -> list: - return list(itertools.chain(*[i.split(",") for i in lst])) - - def prepend(version: str) -> str: - return ( - f"python_version{''.join(i for i in version if i in valid_specifiers)} '" - f"{''.join(i for i in version if i not in valid_specifiers)}'" - ) - - prepend_and_clean = [ - [prepend(".".join(j.split(".")[:2])) for j in conv_and(i)] for i in ver_parsed - ] - return ( - f"{'(' if len(or_and_split) > 1 else ''}" - f"{') or ('.join([' and '.join(i) for i in prepend_and_clean])}" - f"{')' if len(or_and_split) > 1 else ''}" - ) - - -def handle_dict_attr(proj_name: str, attributes: dict[str, str]) -> str: - def produce_match(sep: str, feat: Optional[str]) -> str: - return f"{sep}{feat}" if feat else "" - - git_lookup = attributes.get("git") - if git_lookup is not None: - rev_lookup = produce_match("#", attributes.get("rev")) - branch_lookup = produce_match("@", attributes.get("branch")) - tag_lookup = produce_match("@", attributes.get("tag")) - - return f"{proj_name} @ git+{git_lookup}{tag_lookup}{branch_lookup}{rev_lookup}" - - version_lookup = attributes.get("version") - path_lookup = attributes.get("path") - if path_lookup is not None: - return f"{proj_name} @ file://{path_lookup}" - url_lookup = attributes.get("url") - if url_lookup is not None: - return f"{proj_name} @ {url_lookup}" - if version_lookup is not None: - markers_lookup = produce_match(";", attributes.get("markers")) - python_lookup = parse_python_constraint(attributes.get("python")) - version_parsed = handle_str_attr(proj_name, version_lookup) - return ( - f"{version_parsed}" - f"{markers_lookup}" - f"{' and ' if python_lookup and markers_lookup else (';' if python_lookup else '')}" - f"{python_lookup}" - ) - else: - raise AssertionError( - ( - f"{proj_name} is not formatted correctly; at" - " minimum provide either a version, url, path or git location for" - " your dependency. " - ) - ) - - -def parse_single_dependency( - proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]] -) -> tuple[Requirement, ...]: - if isinstance(attributes, str): - return (Requirement.parse(handle_str_attr(proj_name, attributes)),) - elif isinstance(attributes, dict): - return (Requirement.parse(handle_dict_attr(proj_name, attributes)),) - elif isinstance(attributes, list): - return tuple([Requirement.parse(handle_dict_attr(proj_name, attr)) for attr in attributes]) - else: - raise AssertionError( - ( - "Error: invalid poetry requirement format. Expected " - " type of requirement attributes to be string," - f"dict, or list, but was of type {type(attributes).__name__}." - ) - ) - - -def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement]: - parsed = toml.loads(toml_contents) - try: - poetry_vals = parsed["tool"]["poetry"] - except KeyError: - raise KeyError( - ( - f"No section `tool.poetry` found in {file_path}, which" - "is loaded by Pants from a `poetry_requirements` macro. " - "Did you mean to set up Poetry?" - ) - ) - dependencies = poetry_vals.get("dependencies", {}) - dev_dependencies = poetry_vals.get("dev-dependencies", {}) - if not dependencies and not dev_dependencies: - logger.warning( - ( - "No requirements defined in poetry.tools.dependencies and" - f" poetry.tools.dev-dependencies in {file_path}, which is loaded by Pants" - " from a poetry_requirements macro. Did you mean to populate these" - " with requirements?" - ) - ) - - return set( - itertools.chain.from_iterable( - parse_single_dependency(proj, attr) - for proj, attr in {**dependencies, **dev_dependencies}.items() - ) - ) diff --git a/src/python/pants/backend/python/macros/poetry_project_test.py b/src/python/pants/backend/python/macros/poetry_project_test.py deleted file mode 100644 index fe688a0e4d7..00000000000 --- a/src/python/pants/backend/python/macros/poetry_project_test.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import pytest -from pkg_resources import Requirement - -from pants.backend.python.macros.poetry_requirements import ( - get_max_caret, - get_max_tilde, - handle_dict_attr, - handle_str_attr, - parse_pyproject_toml, - parse_single_dependency, -) - - -@pytest.mark.parametrize( - "test, exp", - [ - ("1.2.3", "2.0.0"), - ("1.2", "2.0.0"), - ("1", "2.0.0"), - ("0.2.3", "0.3.0"), - ("0.0.3", "0.0.4"), - ("0.0", "0.1.0"), - ("0", "1.0.0"), - ], -) -def test_caret(test, exp) -> None: - assert get_max_caret("", test) == exp - - -""" -def test_max_caret_1() -> None: - assert get_max_caret("", "1.2.3") == "2.0.0" - - -def test_max_caret_2() -> None: - assert get_max_caret("", "1.2") == "2.0.0" - - -def test_max_caret_3() -> None: - assert get_max_caret("", "1") == "2.0.0" - - -def test_max_caret_4() -> None: - assert get_max_caret("", "0.2.3") == "0.3.0" - - -def test_max_caret_5() -> None: - assert get_max_caret("", "0.0.3") == "0.0.4" - - -def test_max_caret_6() -> None: - assert get_max_caret("", "0.0") == "0.1.0" - - -def test_max_caret_7() -> None: - assert get_max_caret("", "0") == "1.0.0" - -""" - - -@pytest.mark.parametrize( - "test, exp", [("1.2.3", "1.3.0"), ("1.2", "1.3.0"), ("1", "2.0.0"), ("0", "1.0.0")] -) -def test_max_tilde(test, exp) -> None: - assert get_max_tilde("", test) == exp - - -""" -def test_max_tilde_1() -> None: - assert get_max_tilde("", "1.2.3") == "1.3.0" - - -def test_max_tilde_2() -> None: - assert get_max_tilde("", "1.2") == "1.3.0" - - -def test_max_tilde_3() -> None: - assert get_max_tilde("", "1") == "2.0.0" - - -def test_max_tilde_4() -> None: - assert get_max_tilde("", "0") == "1.0.0" -""" - - -@pytest.mark.parametrize( - "test, exp", - [ - ("~1.2.3", ">=1.2.3,<1.3.0"), - ("^1.2.3", ">=1.2.3,<2.0.0"), - ("~=1.2.3", "~=1.2.3"), - ("1.2.3", "==1.2.3"), - (">1.2.3", ">1.2.3"), - ("~1.2, !=1.2.10", ">=1.2,<1.3.0,!=1.2.10"), - ], -) -def test_handle_str(test, exp) -> None: - assert handle_str_attr("foo", test) == f"foo {exp}" - - -""" -def test_handle_str_tilde() -> None: - assert handle_str_attr("foo", "~1.2.3") == "foo >=1.2.3,<1.3.0" - - -def test_handle_str_caret() -> None: - assert handle_str_attr("foo", "^1.2.3") == "foo >=1.2.3,<2.0.0" - - -def test_handle_compat_operator() -> None: - assert handle_str_attr("foo", "~=1.2.3") == "foo ~=1.2.3" - - -def test_handle_no_operator() -> None: - assert handle_str_attr("foo", "1.2.3") == "foo ==1.2.3" - - -def test_handle_one_char_operator() -> None: - assert handle_str_attr("foo", ">1.2.3") == "foo >1.2.3" - - -def test_handle_multiple_reqs() -> None: - assert handle_str_attr("foo", "~1.2, !=1.2.10") == "foo >=1.2,<1.3.0,!=1.2.10" - -""" - - -def test_handle_git() -> None: - def test_handle_git_basic() -> None: - attr = {"git": "https://github.com/requests/requests.git"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git" - ) - - def test_handle_git_branch() -> None: - attr = {"git": "https://github.com/requests/requests.git", "branch": "main"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git@main" - ) - - def test_handle_git_tag() -> None: - attr = {"git": "https://github.com/requests/requests.git", "tag": "v1.1.1"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git@v1.1.1" - ) - - def test_handle_git_revision() -> None: - attr = {"git": "https://github.com/requests/requests.git", "rev": "1a2b3c4d"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git#1a2b3c4d" - ) - - -def test_handle_path_arg() -> None: - attr = {"path": "../../my_py_proj.whl"} - assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ file://../../my_py_proj.whl" - - -def test_handle_url_arg() -> None: - attr = {"url": "https://my-site.com/mydep.whl"} - assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ https://my-site.com/mydep.whl" - - -def test_version_only() -> None: - attr = {"version": "1.2.3"} - assert handle_dict_attr("foo", attr) == "foo ==1.2.3" - - -def test_py_constraints() -> None: - def test_py_constraint_single() -> None: - attr = {"version": "1.2.3", "python": "3.6"} - assert handle_dict_attr("foo", attr) == "foo ==1.2.3;python_version == '3.6'" - - def test_py_constraint_or() -> None: - attr = {"version": "1.2.3", "python": "3.6 || 3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version == '3.6') or (python_version == '3.7')" - ) - - def test_py_constraint_and() -> None: - attr = {"version": "1.2.3", "python": ">3.6,!=3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;python_version > '3.6' and python_version != '3.7'" - ) - - def test_py_constraint_and_or() -> None: - attr = {"version": "1.2.3", "python": ">3.6 || 3.5,3.4"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')" - ) - - def test_py_constraint_tilde_caret_and_or() -> None: - attr = {"version": "1.2.3", "python": "~3.6 || ^3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')" - ) - - def test_multi_version_const() -> None: - lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] - retval = parse_single_dependency("foo", lst_attr) - actual_reqs = ( - Requirement.parse("foo ==1.2.3; python_version == '3.6'"), - Requirement.parse("foo ==1.2.4; python_version == '3.7'"), - ) - assert retval == actual_reqs - - -def test_extended_form() -> None: - toml_black_str = """ - [tool.poetry.dependencies] - [tool.poetry.dependencies.black] - version = "19.10b0" - python = "3.6" - markers = "platform_python_implementation == 'CPython'" - [tool.poetry.dev-dependencies] - """ - retval = parse_pyproject_toml(toml_black_str, "/path/to/file") - actual_req = { - Requirement.parse( - 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' - ) - } - assert retval == actual_req - - -def test_parse_multi_reqs() -> None: - toml_str = """[tool.poetry] - name = "poetry_tinker" - version = "0.1.0" - description = "" - authors = ["Liam Wilson "] - - [tool.poetry.dependencies] - python = "^3.8" - junk = {url = "https://github.com/myrepo/junk.whl"} - poetry = {git = "https://github.com/python-poetry/poetry.git", tag = "v1.1.1"} - requests = {extras = ["security"], version = "^2.25.1", python = ">2.7"} - foo = [{version = ">=1.9", python = "^2.7"},{version = "^2.0", python = "3.4 || 3.5"}] - - [tool.poetry.dependencies.black] - version = "19.10b0" - python = "3.6" - markers = "platform_python_implementation == 'CPython'" - - [tool.poetry.dev-dependencies] - isort = ">=5.5.1,<5.6" - - [build-system] - requires = ["poetry-core>=1.0.0"] - build-backend = "poetry.core.masonry.api" - """ - retval = parse_pyproject_toml(toml_str, "/path/to/file") - actual_reqs = { - Requirement.parse("python<4.0.0,>=3.8"), - Requirement.parse("junk@ https://github.com/myrepo/junk.whl"), - Requirement.parse("poetry@ git+https://github.com/python-poetry/poetry.git@v1.1.1"), - Requirement.parse('requests<3.0.0,>=2.25.1; python_version > "2.7"'), - Requirement.parse('foo>=1.9; python_version >= "2.7" and python_version < "3.0"'), - Requirement.parse('foo<3.0.0,>=2.0; python_version == "3.4" or python_version == "3.5"'), - Requirement.parse( - 'black==19.10b0; platform_python_implementation == "CPython" and python_version == "3.6"' - ), - Requirement.parse("isort<5.6,>=5.5.1"), - } - assert retval == actual_reqs diff --git a/src/python/pants/backend/python/macros/poetry_requirement.py b/src/python/pants/backend/python/macros/poetry_requirement.py deleted file mode 100644 index 8262beade97..00000000000 --- a/src/python/pants/backend/python/macros/poetry_requirement.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). -import os -from pathlib import Path -from typing import Iterable, Mapping, Optional - -from pants.backend.python.macros.poetry_project import parse_pyproject_toml - -# from pants.backend.python.target_types import parse_requirements_file -from pants.base.build_environment import get_buildroot - - -class PoetryRequirements: - """Translates dependencies specified in a pyproject.toml Poetry file to a set of - "python_requirements_library" targets. - - For example, if pyproject.toml contains the following entries under - poetry.tool.dependencies: `foo = ">1"` and `bar = ">2.4"`, - - python_requirement_library( - name="foo", - requirements=["foo>1"], - ) - - python_requirement_library( - name="bar", - requirements=["bar>2.4"], - ) - - See Poetry documentation for correct specification of pyproject.toml: - https://python-poetry.org/docs/pyproject/ - - You may also use the parameter `module_mapping` to teach Pants what modules each of your - requirements provide. For any requirement unspecified, Pants will default to the name of the - requirement. This setting is important for Pants to know how to convert your import - statements back into your dependencies. For example: - - python_requirements( - module_mapping={ - "ansicolors": ["colors"], - "setuptools": ["pkg_resources"], - } - ) - """ - - def __init__(self, parse_context): - self._parse_context = parse_context - - def __call__( - self, - pyproject_toml_relpath: str = "pyproject.toml", - *, - module_mapping: Optional[Mapping[str, Iterable[str]]] = None, - ) -> None: - """ - :param pyproject_toml_relpath: The relpath from this BUILD file to the requirements file. - Defaults to a `requirements.txt` file sibling to the BUILD file. - :param module_mapping: a mapping of requirement names to a list of the modules they provide. - For example, `{"ansicolors": ["colors"]}`. Any unspecified requirements will use the - requirement name as the default module, e.g. "Django" will default to - `modules=["django"]`. - """ - req_file_tgt = self._parse_context.create_object( - "_python_requirements_file", - name=pyproject_toml_relpath.replace(os.path.sep, "_"), - sources=[pyproject_toml_relpath], - ) - requirements_dep = f":{req_file_tgt.name}" - - req_file = Path(get_buildroot(), self._parse_context.rel_path, pyproject_toml_relpath) - requirements = parse_pyproject_toml( - req_file.read_text(), str(req_file.relative_to(get_buildroot())) - ) - for parsed_req in requirements: - req_module_mapping = ( - {parsed_req.project_name: module_mapping[parsed_req.project_name]} - if module_mapping and parsed_req.project_name in module_mapping - else None - ) - self._parse_context.create_object( - "python_requirement_library", - name=parsed_req.project_name, - requirements=[parsed_req], - module_mapping=req_module_mapping, - dependencies=[requirements_dep], - ) diff --git a/src/python/pants/backend/python/macros/poetry_requirement_test.py b/src/python/pants/backend/python/macros/poetry_requirement_test.py deleted file mode 100644 index a878e7ff760..00000000000 --- a/src/python/pants/backend/python/macros/poetry_requirement_test.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). -# Licensed under the Apache License, Version 2.0 (see LICENSE). - -import logging -from textwrap import dedent -from typing import Iterable - -import pytest -from _pytest.logging import LogCaptureFixture -from pkg_resources import Requirement - -from pants.backend.python.macros.poetry_requirements import PoetryRequirements -from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile -from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs -from pants.engine.addresses import Address -from pants.engine.internals.scheduler import ExecutionError -from pants.engine.target import Targets -from pants.testutil.rule_runner import QueryRule, RuleRunner - -logger = logging.getLogger(__name__) - - -@pytest.fixture -def rule_runner() -> RuleRunner: - return RuleRunner( - rules=[QueryRule(Targets, (Specs,))], - target_types=[PythonRequirementLibrary, PythonRequirementsFile], - context_aware_object_factories={"poetry_requirements": PoetryRequirements}, - ) - - -def assert_poetry_requirements( - rule_runner: RuleRunner, - build_file_entry: str, - pyproject_toml: str, - *, - expected_file_dep: PythonRequirementsFile, - expected_targets: Iterable[PythonRequirementLibrary], - pyproject_toml_relpath: str = "pyproject.toml", -) -> None: - rule_runner.add_to_build_file("", f"{build_file_entry}\n") - rule_runner.create_file(pyproject_toml_relpath, pyproject_toml) - targets = rule_runner.request( - Targets, - [Specs(AddressSpecs([DescendantAddresses("")]), FilesystemSpecs([]))], - ) - assert {expected_file_dep, *expected_targets} == set(targets) - - -def test_pyproject_toml(rule_runner: RuleRunner) -> None: - """This tests that we correctly create a new python_requirement_library for each entry in a - pyproject.toml file. - - Note that this just ensures proper targets are created; whether or not all dependencies are - parsed are checked in tests found in poetry_project_test.py. - """ - assert_poetry_requirements( - rule_runner, - "poetry_requirements(module_mapping={'ansicolors': ['colors']})", - dedent( - """\ - [tool.poetry.dependencies] - Django = {version = "3.2", python = "3"} - ansicolors = ">=1.18.0" - Un-Normalized-PROJECT = "1.0.0" - [tool.poetry.dev-dependencies] - """ - ), - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[ - PythonRequirementLibrary( - { - "dependencies": [":pyproject.toml"], - "requirements": [Requirement.parse("ansicolors>=1.18.0")], - "module_mapping": {"ansicolors": ["colors"]}, - }, - address=Address("", target_name="ansicolors"), - ), - PythonRequirementLibrary( - { - "dependencies": [":pyproject.toml"], - "requirements": [Requirement.parse("Django==3.2 ; python_version == '3'")], - }, - address=Address("", target_name="Django"), - ), - PythonRequirementLibrary( - { - "dependencies": [":pyproject.toml"], - "requirements": [Requirement.parse("Un_Normalized_PROJECT == 1.0.0")], - }, - address=Address("", target_name="Un-Normalized-PROJECT"), - ), - ], - ) - - -def test_relpath_override(rule_runner: RuleRunner) -> None: - assert_poetry_requirements( - rule_runner, - "poetry_requirements(pyproject_toml_relpath='subdir/pyproject.toml')", - dedent( - """\ - [tool.poetry.dependencies] - ansicolors = ">=1.18.0" - [tool.poetry.dev-dependencies] - """ - ), - pyproject_toml_relpath="subdir/pyproject.toml", - expected_file_dep=PythonRequirementsFile( - {"sources": ["subdir/pyproject.toml"]}, - address=Address("", target_name="subdir_pyproject.toml"), - ), - expected_targets=[ - PythonRequirementLibrary( - { - "dependencies": [":subdir_pyproject.toml"], - "requirements": [Requirement.parse("ansicolors>=1.18.0")], - }, - address=Address("", target_name="ansicolors"), - ), - ], - ) - - -def test_non_pep440_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: - assert_poetry_requirements( - rule_runner, - "poetry_requirements()", - """ - [tool.poetry.dependencies] - foo = "~r62b" - [tool.poetry.dev-dependencies] - """, - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[ - PythonRequirementLibrary( - { - "dependencies": [":pyproject.toml"], - "requirements": [Requirement.parse("foo =r62b")], - }, - address=Address("", target_name="foo"), - ) - ], - ) - assert "PEP440" in caplog.text - - -def test_no_req_defined_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: - assert_poetry_requirements( - rule_runner, - "poetry_requirements()", - """ - [tool.poetry.dependencies] - [tool.poetry.dev-dependencies] - """, - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[], - ) - assert "No requirements defined" in caplog.text - - -def test_bad_dict_format(rule_runner: RuleRunner) -> None: - with pytest.raises(ExecutionError) as exc: - assert_poetry_requirements( - rule_runner, - "poetry_requirements()", - """ - [tool.poetry.dependencies] - foo = {bad_req = "test"} - [tool.poetry.dev-dependencies] - """, - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[], - ) - assert "not formatted correctly; at" in str(exc.value) - - -def test_bad_req_type(rule_runner: RuleRunner) -> None: - with pytest.raises(ExecutionError) as exc: - assert_poetry_requirements( - rule_runner, - "poetry_requirements()", - """ - [tool.poetry.dependencies] - foo = 4 - [tool.poetry.dev-dependencies] - """, - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[], - ) - assert "was of type int" in str(exc.value) - - -def test_no_tool_poetry(rule_runner: RuleRunner) -> None: - with pytest.raises(ExecutionError) as exc: - assert_poetry_requirements( - rule_runner, - "poetry_requirements()", - """ - foo = 4 - """, - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[], - ) - assert "`tool.poetry` found in pyproject.toml" in str(exc.value) From c11b7f6652cc138ab383641f5da264ba9649ef2f Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 24 Jun 2021 02:53:19 -0700 Subject: [PATCH 06/11] Refactoring and clean up per code review --- .../python/macros/poetry_requirements.py | 86 +++++----- .../python/macros/poetry_requirements_test.py | 156 +++++++----------- 2 files changed, 99 insertions(+), 143 deletions(-) diff --git a/src/python/pants/backend/python/macros/poetry_requirements.py b/src/python/pants/backend/python/macros/poetry_requirements.py index 46699a9b933..167e761ccba 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements.py +++ b/src/python/pants/backend/python/macros/poetry_requirements.py @@ -18,89 +18,80 @@ logger = logging.getLogger(__name__) -def get_max_caret(proj_name: str, version: str) -> str: - major = "0" - minor = "0" - micro = "0" +def get_max_caret(proj_name: str, version: str, req: str, fp: str) -> str: + major = 0 + minor = 0 + micro = 0 try: parsed_version = Version(version) except InvalidVersion: - logger.warning( - f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" - f" will be left as >={version},<{version}" + raise InvalidVersion( + f"Failed to parse requirement {req} in {fp} loaded by the poetry_requirements macro.\n\nIf you believe this requirement is valid, consider opening an issue at https://github.com/pantsbuild/pants/issues so that we can update Pants's Poetry macro to support this." ) - return version if parsed_version.major != 0: - major = str(parsed_version.major + 1) + major = parsed_version.major + 1 elif parsed_version.minor != 0: - minor = str(parsed_version.minor + 1) + minor = parsed_version.minor + 1 elif parsed_version.micro != 0: - micro = str(parsed_version.micro + 1) + micro = parsed_version.micro + 1 else: base_len = len(parsed_version.base_version.split(".")) if base_len >= 3: - micro = "1" + micro = 1 elif base_len == 2: - minor = "1" + minor = 1 elif base_len == 1: - major = "1" + major = 1 return f"{major}.{minor}.{micro}" -def get_max_tilde(proj_name: str, version: str) -> str: - major = "0" - minor = "0" - micro = "0" +def get_max_tilde(proj_name: str, version: str, req: str, fp: str) -> str: + major = 0 + minor = 0 try: parsed_version = Version(version) except InvalidVersion: - logger.warning( - f"Warning: version {version} for {proj_name} is not PEP440-compliant; this requirement" - f" will be parsed as >={version},<{version}" + raise InvalidVersion( + f'Failed to parse requirement {proj_name} = "{req}" in {fp} loaded by the poetry_requirements macro.\n\nIf you believe this requirement is valid, consider opening an issue at https://github.com/pantsbuild/pants/issues so that we can update Pants\'s Poetry macro to support this.' ) - return version base_len = len(parsed_version.base_version.split(".")) if base_len >= 2: - minor = str(parsed_version.minor + 1) - major = str(parsed_version.major) + minor = parsed_version.minor + 1 + major = parsed_version.major elif base_len == 1: - major = str(parsed_version.major + 1) + major = parsed_version.major + 1 - return f"{major}.{minor}.{micro}" + return f"{major}.{minor}.0" -def handle_str_attr(proj_name: str, attributes: str) -> str: - # kwarg for parse_python_constraint +def parse_str_version(proj_name: str, attributes: str, fp: str) -> str: valid_specifiers = "<>!~=" pep440_reqs = [] - comma_split_reqs = [i.strip() for i in attributes.split(",")] + comma_split_reqs = (i.strip() for i in attributes.split(",")) for req in comma_split_reqs: if req[0] == "^": - max_ver = get_max_caret(proj_name, req[1:]) + max_ver = get_max_caret(proj_name, req[1:], req, fp) min_ver = req[1:] pep440_reqs.append(f">={min_ver},<{max_ver}") # ~= is an acceptable default operator; however, ~ is not, and IS NOT the same as ~= elif req[0] == "~" and req[1] != "=": - max_ver = get_max_tilde(proj_name, req[1:]) + max_ver = get_max_tilde(proj_name, req[1:], req, fp) min_ver = req[1:] pep440_reqs.append(f">={min_ver},<{max_ver}") else: - if req[0] not in valid_specifiers: - pep440_reqs.append(f"=={req}") - else: - pep440_reqs.append(req) + pep440_reqs.append(req if req[0] in valid_specifiers else f"=={req}") return f"{proj_name} {','.join(pep440_reqs)}" -def parse_python_constraint(constr: str | None) -> str: +def parse_python_constraint(constr: str | None, fp: str) -> str: if constr is None: return "" valid_specifiers = "<>!~= " or_and_split = [[j.strip() for j in i.split(",")] for i in constr.split("||")] - ver_parsed = [[handle_str_attr("", j) for j in i] for i in or_and_split] + ver_parsed = [[parse_str_version("", j, fp) for j in i] for i in or_and_split] def conv_and(lst: list[str]) -> list: return list(itertools.chain(*[i.split(",") for i in lst])) @@ -121,7 +112,7 @@ def prepend(version: str) -> str: ) -def handle_dict_attr(proj_name: str, attributes: dict[str, str]) -> str: +def handle_dict_attr(proj_name: str, attributes: dict[str, str], fp: str) -> str: def produce_match(sep: str, feat: Optional[str]) -> str: return f"{sep}{feat}" if feat else "" @@ -142,8 +133,8 @@ def produce_match(sep: str, feat: Optional[str]) -> str: return f"{proj_name} @ {url_lookup}" if version_lookup is not None: markers_lookup = produce_match(";", attributes.get("markers")) - python_lookup = parse_python_constraint(attributes.get("python")) - version_parsed = handle_str_attr(proj_name, version_lookup) + python_lookup = parse_python_constraint(attributes.get("python"), fp) + version_parsed = parse_str_version(proj_name, version_lookup, fp) return ( f"{version_parsed}" f"{markers_lookup}" @@ -161,14 +152,19 @@ def produce_match(sep: str, feat: Optional[str]) -> str: def parse_single_dependency( - proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]] + proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]], fp: str ) -> tuple[Requirement, ...]: if isinstance(attributes, str): - return (Requirement.parse(handle_str_attr(proj_name, attributes)),) + # E.g. `foo = "~1.1~'. + return (Requirement.parse(parse_str_version(proj_name, attributes, fp)),) elif isinstance(attributes, dict): - return (Requirement.parse(handle_dict_attr(proj_name, attributes)),) + # E.g. `foo = {version = "~1.1"}`. + return (Requirement.parse(handle_dict_attr(proj_name, attributes, fp)),) elif isinstance(attributes, list): - return tuple([Requirement.parse(handle_dict_attr(proj_name, attr)) for attr in attributes]) + # E.g. ` foo = [{version = "1.1","python" = "2.7"}, {version = "1.1","python" = "2.7"}] + return tuple( + Requirement.parse(handle_dict_attr(proj_name, attr, fp)) for attr in attributes + ) else: raise AssertionError( ( @@ -205,7 +201,7 @@ def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement] return set( itertools.chain.from_iterable( - parse_single_dependency(proj, attr) + parse_single_dependency(proj, attr, file_path) for proj, attr in {**dependencies, **dev_dependencies}.items() ) ) diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py index 126f45334cb..1ea541e20e9 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements_test.py +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -2,10 +2,9 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from textwrap import dedent -from typing import Iterable +from typing import Any, Dict, Iterable import pytest -from _pytest.logging import LogCaptureFixture from pkg_resources import Requirement from pants.backend.python.macros.poetry_requirements import ( @@ -13,9 +12,9 @@ get_max_caret, get_max_tilde, handle_dict_attr, - handle_str_attr, parse_pyproject_toml, parse_single_dependency, + parse_str_version, ) from pants.backend.python.target_types import PythonRequirementLibrary, PythonRequirementsFile from pants.base.specs import AddressSpecs, DescendantAddresses, FilesystemSpecs, Specs @@ -38,14 +37,14 @@ ], ) def test_caret(test, exp) -> None: - assert get_max_caret("", test) == exp + assert get_max_caret("", test, "", "") == exp @pytest.mark.parametrize( "test, exp", [("1.2.3", "1.3.0"), ("1.2", "1.3.0"), ("1", "2.0.0"), ("0", "1.0.0")] ) def test_max_tilde(test, exp) -> None: - assert get_max_tilde("", test) == exp + assert get_max_tilde("", test, "", "") == exp @pytest.mark.parametrize( @@ -60,95 +59,63 @@ def test_max_tilde(test, exp) -> None: ], ) def test_handle_str(test, exp) -> None: - assert handle_str_attr("foo", test) == f"foo {exp}" + assert parse_str_version("foo", test, "") == f"foo {exp}" def test_handle_git() -> None: - def test_handle_git_basic() -> None: - attr = {"git": "https://github.com/requests/requests.git"} + def assert_git(extra_opts: Dict[str, str], suffix: str) -> None: + attr = {"git": "https://github.com/requests/requests.git", **extra_opts} assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git" + handle_dict_attr("requests", attr, "") + == f"requests @ git+https://github.com/requests/requests.git{suffix}" ) - def test_handle_git_branch() -> None: - attr = {"git": "https://github.com/requests/requests.git", "branch": "main"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git@main" - ) - - def test_handle_git_tag() -> None: - attr = {"git": "https://github.com/requests/requests.git", "tag": "v1.1.1"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git@v1.1.1" - ) - - def test_handle_git_revision() -> None: - attr = {"git": "https://github.com/requests/requests.git", "rev": "1a2b3c4d"} - assert ( - handle_dict_attr("requests", attr) - == "requests @ git+https://github.com/requests/requests.git#1a2b3c4d" - ) + assert_git({}, "") + assert_git({"branch": "main"}, "@main") + assert_git({"tag": "v1.1.1"}, "@v1.1.1") + assert_git({"rev": "1a2b3c4d"}, "#1a2b3c4d") def test_handle_path_arg() -> None: attr = {"path": "../../my_py_proj.whl"} - assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ file://../../my_py_proj.whl" + assert handle_dict_attr("my_py_proj", attr, "") == "my_py_proj @ file://../../my_py_proj.whl" def test_handle_url_arg() -> None: attr = {"url": "https://my-site.com/mydep.whl"} - assert handle_dict_attr("my_py_proj", attr) == "my_py_proj @ https://my-site.com/mydep.whl" + assert handle_dict_attr("my_py_proj", attr, "") == "my_py_proj @ https://my-site.com/mydep.whl" def test_version_only() -> None: attr = {"version": "1.2.3"} - assert handle_dict_attr("foo", attr) == "foo ==1.2.3" + assert handle_dict_attr("foo", attr, "") == "foo ==1.2.3" def test_py_constraints() -> None: - def test_py_constraint_single() -> None: - attr = {"version": "1.2.3", "python": "3.6"} - assert handle_dict_attr("foo", attr) == "foo ==1.2.3;python_version == '3.6'" - - def test_py_constraint_or() -> None: - attr = {"version": "1.2.3", "python": "3.6 || 3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version == '3.6') or (python_version == '3.7')" - ) - - def test_py_constraint_and() -> None: - attr = {"version": "1.2.3", "python": ">3.6,!=3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;python_version > '3.6' and python_version != '3.7'" - ) - - def test_py_constraint_and_or() -> None: - attr = {"version": "1.2.3", "python": ">3.6 || 3.5,3.4"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')" - ) - - def test_py_constraint_tilde_caret_and_or() -> None: - attr = {"version": "1.2.3", "python": "~3.6 || ^3.7"} - assert ( - handle_dict_attr("foo", attr) - == "foo ==1.2.3;(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')" - ) + def assert_py_constraints(py_req: str, suffix: str) -> None: + attr = {"version": "1.2.3", "python": py_req} + assert handle_dict_attr("foo", attr, "") == f"foo ==1.2.3;{suffix}" + + assert_py_constraints("3.6", "python_version == '3.6'") + assert_py_constraints("3.6 || 3.7", "(python_version == '3.6') or (python_version == '3.7')") + assert_py_constraints(">3.6,!=3.7", "python_version > '3.6' and python_version != '3.7'") + assert_py_constraints( + ">3.6 || 3.5,3.4", + "(python_version > '3.6') or (python_version == '3.5' and python_version == '3.4')", + ) + assert_py_constraints( + "~3.6 || ^3.7", + "(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')", + ) - def test_multi_version_const() -> None: - lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] - retval = parse_single_dependency("foo", lst_attr) - actual_reqs = ( - Requirement.parse("foo ==1.2.3; python_version == '3.6'"), - Requirement.parse("foo ==1.2.4; python_version == '3.7'"), - ) - assert retval == actual_reqs +def test_multi_version_const() -> None: + lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] + retval = parse_single_dependency("foo", lst_attr, "") + actual_reqs = ( + Requirement.parse("foo ==1.2.3; python_version == '3.6'"), + Requirement.parse("foo ==1.2.4; python_version == '3.7'"), + ) + assert retval == actual_reqs def test_extended_form() -> None: @@ -252,9 +219,9 @@ def test_pyproject_toml(rule_runner: RuleRunner) -> None: """\ [tool.poetry.dependencies] Django = {version = "3.2", python = "3"} - ansicolors = ">=1.18.0" Un-Normalized-PROJECT = "1.0.0" [tool.poetry.dev-dependencies] + ansicolors = ">=1.18.0" """ ), expected_file_dep=PythonRequirementsFile( @@ -316,33 +283,26 @@ def test_relpath_override(rule_runner: RuleRunner) -> None: ) -def test_non_pep440_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: - assert_poetry_requirements( - rule_runner, - "poetry_requirements()", - """ - [tool.poetry.dependencies] - foo = "~r62b" - [tool.poetry.dev-dependencies] - """, - expected_file_dep=PythonRequirementsFile( - {"sources": ["pyproject.toml"]}, - address=Address("", target_name="pyproject.toml"), - ), - expected_targets=[ - PythonRequirementLibrary( - { - "dependencies": [":pyproject.toml"], - "requirements": [Requirement.parse("foo =r62b")], - }, - address=Address("", target_name="foo"), - ) - ], - ) - assert "PEP440" in caplog.text +def test_non_pep440_error(rule_runner: RuleRunner, caplog: Any) -> None: + with pytest.raises(ExecutionError) as exc: + assert_poetry_requirements( + rule_runner, + "poetry_requirements()", + """ + [tool.poetry.dependencies] + foo = "~r62b" + [tool.poetry.dev-dependencies] + """, + expected_file_dep=PythonRequirementsFile( + {"sources": ["pyproject.toml"]}, + address=Address("", target_name="pyproject.toml"), + ), + expected_targets=[], + ) + assert 'Failed to parse requirement foo = "~r62b" in pyproject.toml' in str(exc.value) -def test_no_req_defined_warning(rule_runner: RuleRunner, caplog: LogCaptureFixture) -> None: +def test_no_req_defined_warning(rule_runner: RuleRunner, caplog: Any) -> None: assert_poetry_requirements( rule_runner, "poetry_requirements()", From 4f23f7ca5261b74b6a30d406d0fb0b13e4b00e8f Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 24 Jun 2021 03:22:27 -0700 Subject: [PATCH 07/11] Formatting # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../pants/backend/python/macros/poetry_requirements_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py index 1ea541e20e9..c64612634e3 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements_test.py +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -108,6 +108,7 @@ def assert_py_constraints(py_req: str, suffix: str) -> None: "(python_version >= '3.6' and python_version< '3.7') or (python_version >= '3.7' and python_version< '4.0')", ) + def test_multi_version_const() -> None: lst_attr = [{"version": "1.2.3", "python": "3.6"}, {"version": "1.2.4", "python": "3.7"}] retval = parse_single_dependency("foo", lst_attr, "") From ec16f576a4ecab0e93bfb0b43eec24033da3606c Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 24 Jun 2021 10:51:47 -0700 Subject: [PATCH 08/11] Stu 's fix # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/option/global_options.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index d1c05b65968..ff5a998f8b7 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -1511,6 +1511,8 @@ def compute_pantsd_invalidation_globs( # macros should be adapted to allow this dependency to be automatically detected. "requirements.txt", "3rdparty/**/requirements.txt", + "pyproject.toml", + "3rdparty/**/requirements.txt", *bootstrap_options.pantsd_invalidation_globs, ) ) From 6843e04e6e6fbb075f6983abf6ef7795b51a3f97 Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 24 Jun 2021 13:19:16 -0700 Subject: [PATCH 09/11] Refactoring and more tests for prereleases # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- .../python/macros/poetry_requirements.py | 47 +++++++++---------- .../python/macros/poetry_requirements_test.py | 24 ++++++++-- 2 files changed, 43 insertions(+), 28 deletions(-) diff --git a/src/python/pants/backend/python/macros/poetry_requirements.py b/src/python/pants/backend/python/macros/poetry_requirements.py index 167e761ccba..e00e496dac6 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements.py +++ b/src/python/pants/backend/python/macros/poetry_requirements.py @@ -18,18 +18,11 @@ logger = logging.getLogger(__name__) -def get_max_caret(proj_name: str, version: str, req: str, fp: str) -> str: +def get_max_caret(parsed_version: Version) -> str: major = 0 minor = 0 micro = 0 - try: - parsed_version = Version(version) - except InvalidVersion: - raise InvalidVersion( - f"Failed to parse requirement {req} in {fp} loaded by the poetry_requirements macro.\n\nIf you believe this requirement is valid, consider opening an issue at https://github.com/pantsbuild/pants/issues so that we can update Pants's Poetry macro to support this." - ) - if parsed_version.major != 0: major = parsed_version.major + 1 elif parsed_version.minor != 0: @@ -48,21 +41,15 @@ def get_max_caret(proj_name: str, version: str, req: str, fp: str) -> str: return f"{major}.{minor}.{micro}" -def get_max_tilde(proj_name: str, version: str, req: str, fp: str) -> str: +def get_max_tilde(parsed_version: Version) -> str: major = 0 minor = 0 - try: - parsed_version = Version(version) - except InvalidVersion: - raise InvalidVersion( - f'Failed to parse requirement {proj_name} = "{req}" in {fp} loaded by the poetry_requirements macro.\n\nIf you believe this requirement is valid, consider opening an issue at https://github.com/pantsbuild/pants/issues so that we can update Pants\'s Poetry macro to support this.' - ) base_len = len(parsed_version.base_version.split(".")) if base_len >= 2: - minor = parsed_version.minor + 1 - major = parsed_version.major + minor = int(str(parsed_version.minor)) + 1 + major = int(str(parsed_version.major)) elif base_len == 1: - major = parsed_version.major + 1 + major = int(str(parsed_version.major)) + 1 return f"{major}.{minor}.0" @@ -72,14 +59,24 @@ def parse_str_version(proj_name: str, attributes: str, fp: str) -> str: pep440_reqs = [] comma_split_reqs = (i.strip() for i in attributes.split(",")) for req in comma_split_reqs: - if req[0] == "^": - max_ver = get_max_caret(proj_name, req[1:], req, fp) - min_ver = req[1:] - pep440_reqs.append(f">={min_ver},<{max_ver}") + is_caret = req[0] == "^" # ~= is an acceptable default operator; however, ~ is not, and IS NOT the same as ~= - elif req[0] == "~" and req[1] != "=": - max_ver = get_max_tilde(proj_name, req[1:], req, fp) - min_ver = req[1:] + is_tilde = req[0] == "~" and req[1] != "=" + if is_caret or is_tilde: + try: + parsed_version = Version(req[1:]) + except InvalidVersion: + raise InvalidVersion( + ( + f'Failed to parse requirement {proj_name} = "{req}" in {fp}' + "loaded by the poetry_requirements macro.\n\nIf you believe this requirement is " + "valid, consider opening an issue at https://github.com/pantsbuild/pants/issues" + "so that we can update Pants's Poetry macro to support this." + ) + ) + + max_ver = get_max_caret(parsed_version) if is_caret else get_max_tilde(parsed_version) + min_ver = f"{parsed_version.base_version}" pep440_reqs.append(f">={min_ver},<{max_ver}") else: pep440_reqs.append(req if req[0] in valid_specifiers else f"=={req}") diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py index c64612634e3..593618a1312 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements_test.py +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -5,6 +5,7 @@ from typing import Any, Dict, Iterable import pytest +from packaging.version import Version from pkg_resources import Requirement from pants.backend.python.macros.poetry_requirements import ( @@ -27,6 +28,10 @@ @pytest.mark.parametrize( "test, exp", [ + ("1.0.0-rc0", "2.0.0"), + ("1.2.3.dev0", "2.0.0"), + ("1.2.3-dev0", "2.0.0"), + ("1.2.3dev0", "2.0.0"), ("1.2.3", "2.0.0"), ("1.2", "2.0.0"), ("1", "2.0.0"), @@ -37,19 +42,32 @@ ], ) def test_caret(test, exp) -> None: - assert get_max_caret("", test, "", "") == exp + version = Version(test) + assert get_max_caret(version) == exp @pytest.mark.parametrize( - "test, exp", [("1.2.3", "1.3.0"), ("1.2", "1.3.0"), ("1", "2.0.0"), ("0", "1.0.0")] + "test, exp", + [ + ("1.2.3", "1.3.0"), + ("1.2", "1.3.0"), + ("1", "2.0.0"), + ("0", "1.0.0"), + ("1.2.3.rc1", "1.3.0"), + ("1.2.3rc1", "1.3.0"), + ("1.2.3-rc1", "1.3.0"), + ], ) def test_max_tilde(test, exp) -> None: - assert get_max_tilde("", test, "", "") == exp + version = Version(test) + assert get_max_tilde(version) == exp @pytest.mark.parametrize( "test, exp", [ + ("~1.0.0rc0", ">=1.0.0,<1.1.0"), + ("^1.0.0rc0", ">=1.0.0,<2.0.0"), ("~1.2.3", ">=1.2.3,<1.3.0"), ("^1.2.3", ">=1.2.3,<2.0.0"), ("~=1.2.3", "~=1.2.3"), From 85c075452689c6a42bd79dd5214d35c6e22ead2f Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 24 Jun 2021 18:30:59 -0700 Subject: [PATCH 10/11] Update src/python/pants/option/global_options.py Co-authored-by: Eric Arellano <14852634+Eric-Arellano@users.noreply.github.com> --- src/python/pants/option/global_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index ff5a998f8b7..c398815f065 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -1512,7 +1512,7 @@ def compute_pantsd_invalidation_globs( "requirements.txt", "3rdparty/**/requirements.txt", "pyproject.toml", - "3rdparty/**/requirements.txt", + "3rdparty/**/pyproject.toml", *bootstrap_options.pantsd_invalidation_globs, ) ) From a8ada22bb22e1333f223f8729e564ea53e6c8c9e Mon Sep 17 00:00:00 2001 From: Liam Wilson Date: Thu, 24 Jun 2021 19:10:37 -0700 Subject: [PATCH 11/11] Minimum versions for prereleases is now the prerelease # Building wheels and fs_util will be skipped. Delete if not intended. [ci skip-build-wheels] --- src/python/pants/backend/python/macros/poetry_requirements.py | 2 +- .../pants/backend/python/macros/poetry_requirements_test.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/pants/backend/python/macros/poetry_requirements.py b/src/python/pants/backend/python/macros/poetry_requirements.py index e00e496dac6..b91c7303339 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements.py +++ b/src/python/pants/backend/python/macros/poetry_requirements.py @@ -76,7 +76,7 @@ def parse_str_version(proj_name: str, attributes: str, fp: str) -> str: ) max_ver = get_max_caret(parsed_version) if is_caret else get_max_tilde(parsed_version) - min_ver = f"{parsed_version.base_version}" + min_ver = f"{parsed_version.public}" pep440_reqs.append(f">={min_ver},<{max_ver}") else: pep440_reqs.append(req if req[0] in valid_specifiers else f"=={req}") diff --git a/src/python/pants/backend/python/macros/poetry_requirements_test.py b/src/python/pants/backend/python/macros/poetry_requirements_test.py index 593618a1312..c3e80250d32 100644 --- a/src/python/pants/backend/python/macros/poetry_requirements_test.py +++ b/src/python/pants/backend/python/macros/poetry_requirements_test.py @@ -66,8 +66,8 @@ def test_max_tilde(test, exp) -> None: @pytest.mark.parametrize( "test, exp", [ - ("~1.0.0rc0", ">=1.0.0,<1.1.0"), - ("^1.0.0rc0", ">=1.0.0,<2.0.0"), + ("~1.0.0rc0", ">=1.0.0rc0,<1.1.0"), + ("^1.0.0rc0", ">=1.0.0rc0,<2.0.0"), ("~1.2.3", ">=1.2.3,<1.3.0"), ("^1.2.3", ">=1.2.3,<2.0.0"), ("~=1.2.3", "~=1.2.3"),