Fix poetry_requirements: ignore internal projects. (#12280)

Previously dependencies with paths were assumed to be non-Pants
controlled projects and the resulting requirements were not generated
with absolute file URLs. Fix both issues by assuming all paths that
point to directories inside the build root are Pants controlled
projects and treating the rest as absolute file URL requirements.

Fixes #12272

[ci skip-rust]
[ci skip-build-wheels]

src/python/pants/backend/python/macros/poetry_requirements.py

@@ -6,14 +6,15 @@
 import itertools
 import logging
 import os
-from pathlib import Path
-from typing import Any, Iterable, Mapping, Optional
+from dataclasses import dataclass
+from pathlib import Path, PurePath
+from typing import Any, Iterable, Iterator, Mapping, Optional
 
 import toml
 from packaging.version import InvalidVersion, Version
 from pkg_resources import Requirement
 
-from pants.base.build_environment import get_buildroot
+from pants.base.parse_context import ParseContext
 
 logger = logging.getLogger(__name__)
 
@@ -67,12 +68,10 @@ def parse_str_version(proj_name: str, attributes: str, fp: str) -> str:
                 parsed_version = Version(req[1:])
             except InvalidVersion:
                 raise InvalidVersion(
-                    (
-                        f'Failed to parse requirement {proj_name} = "{req}" in {fp}'
-                        "loaded by the poetry_requirements macro.\n\nIf you believe this requirement is "
-                        "valid, consider opening an issue at https://github.com/pantsbuild/pants/issues"
-                        "so that we can update Pants's Poetry macro to support this."
-                    )
+                    f'Failed to parse requirement {proj_name} = "{req}" in {fp} loaded by the '
+                    "poetry_requirements macro.\n\nIf you believe this requirement is valid, "
+                    "consider opening an issue at https://github.com/pantsbuild/pants/issues so "
+                    "that we can update Pants' Poetry macro to support this."
                 )
 
             max_ver = get_max_caret(parsed_version) if is_caret else get_max_tilde(parsed_version)
@@ -109,7 +108,68 @@ def prepend(version: str) -> str:
     )
 
 
-def handle_dict_attr(proj_name: str, attributes: dict[str, str], fp: str) -> str:
+@dataclass(frozen=True)
+class PyProjectToml:
+    build_root: PurePath
+    toml_relpath: PurePath
+    toml_contents: str
+
+    @classmethod
+    def create(cls, parse_context: ParseContext, pyproject_toml_relpath: str) -> PyProjectToml:
+        build_root = Path(parse_context.build_root)
+        toml_relpath = PurePath(parse_context.rel_path, pyproject_toml_relpath)
+        return cls(
+            build_root=build_root,
+            toml_relpath=toml_relpath,
+            toml_contents=(build_root / toml_relpath).read_text(),
+        )
+
+    def parse(self) -> Mapping[str, Any]:
+        return toml.loads(self.toml_contents)
+
+    def _non_pants_project_abs_path(self, path: Path) -> Path | None:
+        resolved = path.resolve()
+        if resolved.is_file():
+            return resolved
+
+        try:
+            resolved.relative_to(self.build_root)
+        except ValueError:
+            return resolved
+
+        return None
+
+    def non_pants_project_abs_path(self, path: str) -> Path | None:
+        """Determine if the given path represents a non-Pants controlled project.
+
+        If the path points to a file, it's assumed the file is a distribution ( a wheel or sdist)
+        and the absolute path of that file is returned.
+
+        If the path points to a directory and that directory is outside of the build root, it's
+        assumed the directory is the root of a buildable Python project (i.e.: it contains a
+        pyproject.toml or setup.py) and the absolute path of the project is returned.
+
+        Otherwise, `None` is returned since the directory lies inside the build root and is assumed
+        to be a Pants controlled project.
+        """
+        # TODO(John Sirois): This leaves the case where the path is a Python project directory
+        #  inside the build root that the user actually wants Pex / Pip to build. A concrete case
+        #  for this would be a repo where third party is partially handled with vendored exploded
+        #  source distributions. If someone in the wild needs the described case, plumb a
+        #  PoetryRequirements parameter that can list paths to treat as Pants controlled or
+        #  vice-versa.
+        given_path = Path(path)
+        if given_path.is_absolute():
+            return self._non_pants_project_abs_path(given_path)
+        else:
+            return self._non_pants_project_abs_path(
+                Path(self.build_root / self.toml_relpath).parent / given_path
+            )
+
+
+def handle_dict_attr(
+    proj_name: str, attributes: dict[str, str], pyproject_toml: PyProjectToml
+) -> str | None:
     def produce_match(sep: str, feat: Optional[str]) -> str:
         return f"{sep}{feat}" if feat else ""
 
@@ -122,14 +182,23 @@ def produce_match(sep: str, feat: Optional[str]) -> str:
         return f"{proj_name} @ git+{git_lookup}{tag_lookup}{branch_lookup}{rev_lookup}"
 
     version_lookup = attributes.get("version")
+
     path_lookup = attributes.get("path")
     if path_lookup is not None:
-        return f"{proj_name} @ file://{path_lookup}"
+        non_pants_project_abs_path = pyproject_toml.non_pants_project_abs_path(path_lookup)
+        if non_pants_project_abs_path:
+            return f"{proj_name} @ file://{non_pants_project_abs_path}"
+        # An internal path will be handled by normal Pants dependencies and dependency inference;
+        # i.e.: it never represents a third party requirement.
+        return None
+
     url_lookup = attributes.get("url")
     if url_lookup is not None:
         return f"{proj_name} @ {url_lookup}"
+
     if version_lookup is not None:
         markers_lookup = produce_match(";", attributes.get("markers"))
+        fp = str(pyproject_toml.toml_relpath)
         python_lookup = parse_python_constraint(attributes.get("python"), fp)
         version_parsed = parse_str_version(proj_name, version_lookup, fp)
         return (
@@ -138,51 +207,50 @@ def produce_match(sep: str, feat: Optional[str]) -> str:
             f"{' and ' if python_lookup and markers_lookup else (';' if python_lookup else '')}"
             f"{python_lookup}"
         )
-    else:
-        raise AssertionError(
-            (
-                f"{proj_name} is not formatted correctly; at"
-                " minimum provide either a version, url, path or git location for"
-                " your dependency. "
-            )
-        )
+
+    raise AssertionError(
+        f"{proj_name} is not formatted correctly; at minimum provide either a version, url, path "
+        "or git location for your dependency. "
+    )
 
 
 def parse_single_dependency(
-    proj_name: str, attributes: str | dict[str, Any] | list[dict[str, Any]], fp: str
-) -> tuple[Requirement, ...]:
+    proj_name: str,
+    attributes: str | dict[str, Any] | list[dict[str, Any]],
+    pyproject_toml: PyProjectToml,
+) -> Iterator[Requirement]:
     if isinstance(attributes, str):
         # E.g. `foo = "~1.1~'.
-        return (Requirement.parse(parse_str_version(proj_name, attributes, fp)),)
+        yield Requirement.parse(
+            parse_str_version(proj_name, attributes, str(pyproject_toml.toml_relpath))
+        )
     elif isinstance(attributes, dict):
         # E.g. `foo = {version = "~1.1"}`.
-        return (Requirement.parse(handle_dict_attr(proj_name, attributes, fp)),)
+        req_str = handle_dict_attr(proj_name, attributes, pyproject_toml)
+        if req_str:
+            yield Requirement.parse(req_str)
     elif isinstance(attributes, list):
         # E.g. ` foo = [{version = "1.1","python" = "2.7"}, {version = "1.1","python" = "2.7"}]
-        return tuple(
-            Requirement.parse(handle_dict_attr(proj_name, attr, fp)) for attr in attributes
-        )
+        for attr in attributes:
+            req_str = handle_dict_attr(proj_name, attr, pyproject_toml)
+            if req_str:
+                yield Requirement.parse(req_str)
     else:
         raise AssertionError(
-            (
-                "Error: invalid poetry requirement format. Expected "
-                " type of requirement attributes to be string,"
-                f"dict, or list, but was of type {type(attributes).__name__}."
-            )
+            "Error: invalid poetry requirement format. Expected type of requirement attributes to "
+            f"be string, dict, or list, but was of type {type(attributes).__name__}."
         )
 
 
-def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement]:
-    parsed = toml.loads(toml_contents)
+def parse_pyproject_toml(pyproject_toml: PyProjectToml) -> set[Requirement]:
+    parsed = pyproject_toml.parse()
     try:
         poetry_vals = parsed["tool"]["poetry"]
     except KeyError:
         raise KeyError(
-            (
-                f"No section `tool.poetry` found in {file_path}, which"
-                "is loaded by Pants from a `poetry_requirements` macro. "
-                "Did you mean to set up Poetry?"
-            )
+            f"No section `tool.poetry` found in {pyproject_toml.toml_relpath}, which "
+            "is loaded by Pants from a `poetry_requirements` macro. "
+            "Did you mean to set up Poetry?"
         )
     dependencies = poetry_vals.get("dependencies", {})
     # N.B.: The "python" dependency is a special dependency required by Poetry that only serves to
@@ -193,17 +261,15 @@ def parse_pyproject_toml(toml_contents: str, file_path: str) -> set[Requirement]
     dev_dependencies = poetry_vals.get("dev-dependencies", {})
     if not dependencies and not dev_dependencies:
         logger.warning(
-            (
-                "No requirements defined in poetry.tools.dependencies and"
-                f" poetry.tools.dev-dependencies in {file_path}, which is loaded by Pants"
-                " from a poetry_requirements macro. Did you mean to populate these"
-                " with requirements?"
-            )
+            "No requirements defined in poetry.tools.dependencies and "
+            f"poetry.tools.dev-dependencies in {pyproject_toml.toml_relpath}, which is loaded "
+            "by Pants from a poetry_requirements macro. Did you mean to populate these "
+            "with requirements?"
         )
 
     return set(
         itertools.chain.from_iterable(
-            parse_single_dependency(proj, attr, file_path)
+            parse_single_dependency(proj, attr, pyproject_toml)
             for proj, attr in {**dependencies, **dev_dependencies}.items()
         )
     )
@@ -267,9 +333,8 @@ def __call__(
         )
         requirements_dep = f":{req_file_tgt.name}"
 
-        req_file = Path(get_buildroot(), self._parse_context.rel_path, pyproject_toml_relpath)
         requirements = parse_pyproject_toml(
-            req_file.read_text(), str(req_file.relative_to(get_buildroot()))
+            PyProjectToml.create(self._parse_context, pyproject_toml_relpath)
         )
         for parsed_req in requirements:
             proj_name = parsed_req.project_name