diff --git a/cachi2/core/package_managers/pip.py b/cachi2/core/package_managers/pip.py index 08f55027e..acf140762 100644 --- a/cachi2/core/package_managers/pip.py +++ b/cachi2/core/package_managers/pip.py @@ -273,72 +273,78 @@ def _generate_purl_dependency(package: dict[str, Any]) -> str: return purl.to_string() -def _get_pip_metadata(package_dir: RootedPath) -> tuple[str, Optional[str]]: - """ - Attempt to get the name and version of a Pip package. +def _infer_package_name_from_origin_url(package_dir: RootedPath) -> str: + try: + repo_id = get_repo_id(package_dir.root) + except UnsupportedFeature: + raise PackageRejected( + reason="Unable to infer package name from origin URL", + solution=( + "Provide valid metadata in the package files or ensure" + "the git repository has an 'origin' remote with a valid URL." + ), + docs=PIP_METADATA_DOC, + ) + + repo_name = Path(repo_id.parsed_origin_url.path).stem + resolved_name = Path(repo_name).joinpath(package_dir.subpath_from_root) + return canonicalize_name(str(resolved_name).replace("/", "-")).strip("-.") - First, try to parse the setup.py script (if present) and extract name and version - from keyword arguments to the setuptools.setup() call. If either name or version - could not be resolved and there is a setup.cfg file, try to fill in the missing - values from metadata.name and metadata.version in the .cfg file. - :param package_dir: Path to the root directory of a Pip package - :return: Tuple of strings (name, version) +def _extract_metadata_from_config_files( + package_dir: RootedPath, +) -> tuple[Optional[str], Optional[str]]: """ - name = None - version = None + Extract package name and version in the following order. + + 1. pyproject.toml + 2. setup.py + 3. setup.cfg + Note: version is optional in the SBOM, but name is required + """ pyproject_toml = PyProjectTOML(package_dir) + if pyproject_toml.exists(): + log.debug("Checking pyproject.toml for metadata") + name = pyproject_toml.get_name() + version = pyproject_toml.get_version() + + if name: + return name, version + setup_py = SetupPY(package_dir) + if setup_py.exists(): + log.debug("Checking setup.py for metadata") + name = setup_py.get_name() + version = setup_py.get_version() + + if name: + return name, version + setup_cfg = SetupCFG(package_dir) + if setup_cfg.exists(): + log.debug("Checking setup.cfg for metadata") + name = setup_cfg.get_name() + version = setup_cfg.get_version() - if pyproject_toml.exists(): - log.info("Extracting metadata from pyproject.toml") - if pyproject_toml.check_dynamic_version(): - log.warning("Parsing dynamic metadata from pyproject.toml is not supported") + if name: + return name, version - name = pyproject_toml.get_name() - version = pyproject_toml.get_version() + return None, None - if None in (name, version) and setup_py.exists(): - log.info("Filling in missing metadata from setup.py") - name = name or setup_py.get_name() - version = version or setup_py.get_version() - if None in (name, version) and setup_cfg.exists(): - log.info("Filling in missing metadata from setup.cfg") - name = name or setup_cfg.get_name() - version = version or setup_cfg.get_version() +def _get_pip_metadata(package_dir: RootedPath) -> tuple[str, Optional[str]]: + """Attempt to retrieve name and version of a pip package.""" + name, version = _extract_metadata_from_config_files(package_dir) if not name: - log.info("Processing metadata from git repository") - try: - repo_path = get_repo_id(package_dir.root).parsed_origin_url.path.removesuffix(".git") - repo_name = Path(repo_path).name - package_subpath = package_dir.subpath_from_root - - resolved_path = Path(repo_name).joinpath(package_subpath) - normalized_path = canonicalize_name(str(resolved_path).replace("/", "-")) - name = normalized_path.strip("-.") - except UnsupportedFeature: - raise PackageRejected( - reason="Could not take name from the repository origin url", - solution=( - "Please specify package metadata in a way that Cachi2 understands" - " (see the docs)\n" - "or make sure that the directory Cachi2 is processing is a git" - " repository with\n" - "an 'origin' remote in which case Cachi2 will infer the package name from" - " the remote url." - ), - docs=PIP_METADATA_DOC, - ) + name = _infer_package_name_from_origin_url(package_dir) - log.info("Resolved package name: %r", name) + log.info("Resolved name %s for package at %s", name, package_dir) if version: - log.info("Resolved package version: %r", version) + log.info("Resolved version %s for package at %s", version, package_dir) else: - log.warning("Could not resolve package version") + log.warning("Could not resolve version for package at %s", package_dir) return name, version @@ -453,14 +459,6 @@ def get_version(self) -> Optional[str]: log.warning("No project.version in pyproject.toml") return None - def check_dynamic_version(self) -> bool: - """Check if project version is set dynamically.""" - try: - dynamic_properties = self._parsed_toml["project"]["dynamic"] - return "version" in dynamic_properties - except KeyError: - return False - @functools.cached_property def _parsed_toml(self) -> dict[str, Any]: try: diff --git a/tests/unit/package_managers/test_pip.py b/tests/unit/package_managers/test_pip.py index 181cf65ee..ba20cc6d1 100644 --- a/tests/unit/package_managers/test_pip.py +++ b/tests/unit/package_managers/test_pip.py @@ -10,6 +10,7 @@ import pypi_simple import pytest from _pytest.logging import LogCaptureFixture +from git import Repo from cachi2.core.checksum import ChecksumInfo from cachi2.core.errors import ( @@ -24,13 +25,8 @@ from cachi2.core.models.sbom import Component, Property from cachi2.core.package_managers import pip from cachi2.core.rooted_path import PathOutsideRoot, RootedPath -from cachi2.core.scm import RepoID from tests.common_utils import GIT_REF, Symlink, write_file_tree -THIS_MODULE_DIR = Path(__file__).resolve().parent -PKG_DIR = RootedPath("/foo/package_dir") -PKG_DIR_SUBPATH = PKG_DIR.join_within_root("subpath") -MOCK_REPO_ID = RepoID("https://github.com/foolish/bar.git", "abcdef1234") CUSTOM_PYPI_ENDPOINT = "https://my-pypi.org/simple/" @@ -58,116 +54,167 @@ def make_dpi( ) -@pytest.mark.parametrize("toml_exists", [True, False]) -@pytest.mark.parametrize("toml_name", ["name_in_pyproject_toml", None]) -@pytest.mark.parametrize("toml_version", ["version_in_pyproject_toml", None]) -@pytest.mark.parametrize("py_exists", [True, False]) -@pytest.mark.parametrize("py_name", ["name_in_setup_py", None]) -@pytest.mark.parametrize("py_version", ["version_in_setup_py", None]) -@pytest.mark.parametrize("cfg_exists", [True, False]) -@pytest.mark.parametrize("cfg_name", ["name_in_setup_cfg", None]) -@pytest.mark.parametrize("cfg_version", ["version_in_setup_cfg", None]) -@pytest.mark.parametrize("repo_name_with_subpath", ["bar-subpath", None]) -@mock.patch("cachi2.core.package_managers.pip.SetupCFG") -@mock.patch("cachi2.core.package_managers.pip.SetupPY") @mock.patch("cachi2.core.package_managers.pip.PyProjectTOML") -@mock.patch("cachi2.core.package_managers.pip.get_repo_id") -def test_get_pip_metadata( - mock_get_repo_id: mock.Mock, +def test_get_pip_metadata_from_pyproject_toml( mock_pyproject_toml: mock.Mock, + rooted_tmp_path: RootedPath, + caplog: LogCaptureFixture, +) -> None: + pyproject_toml = mock_pyproject_toml.return_value + pyproject_toml.exists.return_value = True + pyproject_toml.get_name.return_value = "foo" + pyproject_toml.get_version.return_value = "0.1.0" + + name, version = pip._get_pip_metadata(rooted_tmp_path) + assert name == "foo" + assert version == "0.1.0" + assert "Checking pyproject.toml for metadata" in caplog.messages + + # check logs + assert f"Resolved name {name} for package at {rooted_tmp_path}" in caplog.messages + assert f"Resolved version {version} for package at {rooted_tmp_path}" in caplog.messages + + +@mock.patch("cachi2.core.package_managers.pip.SetupPY") +def test_get_pip_metadata_from_setup_py( mock_setup_py: mock.Mock, + rooted_tmp_path: RootedPath, + caplog: LogCaptureFixture, +) -> None: + setup_py = mock_setup_py.return_value + setup_py.exists.return_value = True + setup_py.get_name.return_value = "foo" + setup_py.get_version.return_value = "0.1.0" + + name, version = pip._get_pip_metadata(rooted_tmp_path) + assert name == "foo" + assert version == "0.1.0" + + # check logs + assert "Checking setup.py for metadata" in caplog.messages + assert f"Resolved name {name} for package at {rooted_tmp_path}" in caplog.messages + assert f"Resolved version {version} for package at {rooted_tmp_path}" in caplog.messages + + +@mock.patch("cachi2.core.package_managers.pip.SetupCFG") +def test_get_pip_metadata_from_setup_cfg( mock_setup_cfg: mock.Mock, - toml_exists: bool, - toml_name: Optional[str], - toml_version: Optional[str], - py_exists: bool, - py_name: Optional[str], - py_version: Optional[str], - cfg_exists: bool, - cfg_name: Optional[str], - cfg_version: Optional[str], - repo_name_with_subpath: Optional[str], - caplog: pytest.LogCaptureFixture, + rooted_tmp_path: RootedPath, + caplog: LogCaptureFixture, ) -> None: - """ - Test get_pip_metadata() function. - - More thorough tests of pyproject.toml, setup.py and setup.cfg handling are in their respective classes. - """ - if not toml_exists: - toml_name = None - toml_version = None - if not py_exists: - py_name = None - py_version = None - if not cfg_exists: - cfg_name = None - cfg_version = None + setup_cfg = mock_setup_cfg.return_value + setup_cfg.exists.return_value = True + setup_cfg.get_name.return_value = "foo" + setup_cfg.get_version.return_value = "0.1.0" + name, version = pip._get_pip_metadata(rooted_tmp_path) + assert name == "foo" + assert version == "0.1.0" + + # check logs + assert "Checking setup.cfg for metadata" in caplog.messages + assert f"Resolved name {name} for package at {rooted_tmp_path}" in caplog.messages + assert f"Resolved version {version} for package at {rooted_tmp_path}" in caplog.messages + + +@mock.patch("cachi2.core.package_managers.pip.PyProjectTOML") +@mock.patch("cachi2.core.package_managers.pip.SetupCFG") +@mock.patch("cachi2.core.package_managers.pip.SetupPY") +def test_extract_metadata_from_config_files_with_fallbacks( + mock_setup_py: mock.Mock, + mock_setup_cfg: mock.Mock, + mock_pyproject_toml: mock.Mock, + rooted_tmp_path: RootedPath, + caplog: LogCaptureFixture, +) -> None: + # Case 1: Only pyproject.toml exists with name but no version pyproject_toml = mock_pyproject_toml.return_value - pyproject_toml.exists.return_value = toml_exists - pyproject_toml.get_name.return_value = toml_name - pyproject_toml.get_version.return_value = toml_version + pyproject_toml.exists.return_value = True + pyproject_toml.get_name.return_value = "name_from_pyproject_toml" + pyproject_toml.get_version.return_value = None + + setup_cfg = mock_setup_cfg.return_value + setup_cfg.exists.return_value = False setup_py = mock_setup_py.return_value - setup_py.exists.return_value = py_exists - setup_py.get_name.return_value = py_name - setup_py.get_version.return_value = py_version + setup_py.exists.return_value = False - setup_cfg = mock_setup_cfg.return_value - setup_cfg.exists.return_value = cfg_exists - setup_cfg.get_name.return_value = cfg_name - setup_cfg.get_version.return_value = cfg_version + name, version = pip._extract_metadata_from_config_files(rooted_tmp_path) + assert name == "name_from_pyproject_toml" + assert version is None + assert "Checking pyproject.toml for metadata" in caplog.messages - mock_get_repo_id.return_value = MOCK_REPO_ID + # Case 2: pyproject.toml exists but without a name; fallback to setup.py with name and version + pyproject_toml.get_name.return_value = None - expect_name = toml_name or py_name or cfg_name or repo_name_with_subpath - expect_version = toml_version or py_version or cfg_version + setup_py.exists.return_value = True + setup_py.get_name.return_value = "name_from_setup_py" + setup_py.get_version.return_value = "0.1.0" - if expect_name: - name, version = pip._get_pip_metadata(PKG_DIR_SUBPATH) + name, version = pip._extract_metadata_from_config_files(rooted_tmp_path) + assert name == "name_from_setup_py" + assert version == "0.1.0" + assert "Checking setup.py for metadata" in caplog.messages - assert name == expect_name - assert version == expect_version - else: - mock_get_repo_id.side_effect = UnsupportedFeature( - "Cachi2 cannot process repositories that don't have an 'origin' remote" - ) - with pytest.raises(PackageRejected) as exc_info: - pip._get_pip_metadata(PKG_DIR_SUBPATH) - assert str(exc_info.value) == "Could not take name from the repository origin url" - return + # Case 3: Both pyproject.toml and setup.py lack names; fallback to setup.cfg with complete metadata + setup_py.get_name.return_value = None + + setup_cfg.exists.return_value = True + setup_cfg.get_name.return_value = "name_from_setup_cfg" + setup_cfg.get_version.return_value = "0.2.0" + + name, version = pip._extract_metadata_from_config_files(rooted_tmp_path) + assert name == "name_from_setup_cfg" + assert version == "0.2.0" + assert "Checking setup.cfg for metadata" in caplog.messages - assert pyproject_toml.get_name.called == toml_exists - assert pyproject_toml.get_version.called == toml_exists + # Case 4: None of the config files have names, resulting in None, None + setup_cfg.get_name.return_value = None - find_name_in_setup_py = toml_name is None and py_exists - find_version_in_setup_py = toml_version is None and py_exists - find_name_in_setup_cfg = toml_name is None and py_name is None and cfg_exists - find_version_in_setup_cfg = toml_version is None and py_version is None and cfg_exists + name, version = pip._extract_metadata_from_config_files(rooted_tmp_path) + assert name is None + assert version is None - assert setup_py.get_name.called == find_name_in_setup_py - assert setup_py.get_version.called == find_version_in_setup_py - assert setup_cfg.get_name.called == find_name_in_setup_cfg - assert setup_cfg.get_version.called == find_version_in_setup_cfg +@pytest.mark.parametrize( + "origin_exists", + [True, False], +) +@mock.patch("cachi2.core.package_managers.pip.PyProjectTOML") +@mock.patch("cachi2.core.package_managers.pip.SetupPY") +@mock.patch("cachi2.core.package_managers.pip.SetupCFG") +def test_get_pip_metadata_from_remote_origin( + mock_setup_cfg: mock.Mock, + mock_setup_py: mock.Mock, + mock_pyproject_toml: mock.Mock, + origin_exists: bool, + rooted_tmp_path_repo: RootedPath, + caplog: LogCaptureFixture, +) -> None: + pyproject_toml = mock_pyproject_toml.return_value + pyproject_toml.exists.return_value = False - if toml_exists: - assert "Extracting metadata from pyproject.toml" in caplog.text + setup_py = mock_setup_py.return_value + setup_py.exists.return_value = False + + setup_cfg = mock_setup_cfg.return_value + setup_cfg.exists.return_value = False - if find_name_in_setup_py or find_version_in_setup_py: - assert "Filling in missing metadata from setup.py" in caplog.text + if origin_exists: + repo = Repo(rooted_tmp_path_repo) + repo.create_remote("origin", "git@github.com:user/repo.git") - if find_name_in_setup_cfg or find_version_in_setup_cfg: - assert "Filling in missing metadata from setup.cfg" in caplog.text + name, version = pip._get_pip_metadata(rooted_tmp_path_repo) + assert name == "repo" + assert version is None - if not (toml_exists or py_exists or cfg_exists): - assert "Processing metadata from git repository" in caplog.text + assert f"Resolved name repo for package at {rooted_tmp_path_repo}" in caplog.messages + assert f"Could not resolve version for package at {rooted_tmp_path_repo}" in caplog.messages + else: + with pytest.raises(PackageRejected) as exc_info: + pip._get_pip_metadata(rooted_tmp_path_repo) - if expect_name: - assert f"Resolved package name: '{expect_name}'" in caplog.text - if expect_version: - assert f"Resolved package version: '{expect_version}'" in caplog.text + assert str(exc_info.value) == "Unable to infer package name from origin URL" class TestPyprojectTOML: @@ -187,39 +234,6 @@ def _assert_has_logs( for log in expect_logs: assert log.format(tmpdir=tmpdir) in caplog.text - @pytest.mark.parametrize( - "toml_content, expect_logs", - [ - ( - dedent( - """\ - [project] - name = "my-package" - dynamic = ["version", "readme"] - description = "A short description of the package." - license = "MIT" - """ - ), - [ - "Parsing pyproject.toml at '{tmpdir}/pyproject.toml'", - ], - ) - ], - ) - def test_check_dynamic_version( - self, - toml_content: str, - expect_logs: list[str], - rooted_tmp_path: RootedPath, - caplog: pytest.LogCaptureFixture, - ) -> None: - """Test check_dynamic_version() method.""" - pyproject_toml = rooted_tmp_path.join_within_root("pyproject.toml") - pyproject_toml.path.write_text(toml_content) - - assert pip.PyProjectTOML(rooted_tmp_path).check_dynamic_version() - self._assert_has_logs(expect_logs, rooted_tmp_path.path, caplog) - @pytest.mark.parametrize( "toml_content, expect_name, expect_logs", [