diff --git a/conda_lock/pypi_solver.py b/conda_lock/pypi_solver.py index 245db150..08846bb0 100644 --- a/conda_lock/pypi_solver.py +++ b/conda_lock/pypi_solver.py @@ -1,14 +1,16 @@ import re import sys +import warnings from pathlib import Path from posixpath import expandvars -from typing import TYPE_CHECKING, Dict, List, Optional -from urllib.parse import urldefrag, urlparse, urlsplit, urlunsplit +from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple +from urllib.parse import urldefrag, urlsplit, urlunsplit from clikit.api.io.flags import VERY_VERBOSE from clikit.io import ConsoleIO, NullIO from packaging.tags import compatible_tags, cpython_tags, mac_platforms +from packaging.version import Version from conda_lock.interfaces.vendored_poetry import ( Chooser, @@ -39,7 +41,6 @@ if TYPE_CHECKING: from packaging.tags import Tag - # NB: in principle these depend on the glibc on the machine creating the conda env. # We use tags supported by manylinux Docker images, which are likely the most common # in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images. @@ -55,6 +56,12 @@ class PlatformEnv(Env): Fake poetry Env to match PyPI distributions to the target conda environment """ + _sys_platform: Literal["darwin", "linux", "win32"] + _platform_system: Literal["Darwin", "Linux", "Windows"] + _os_name: Literal["posix", "nt"] + _platforms: List[str] + _python_version: Tuple[int, ...] + def __init__( self, python_version: str, @@ -67,33 +74,12 @@ def __init__( arch = "x86_64" if system == "linux": - # We use MANYLINUX_TAGS but only go up to the latest supported version - # as provided by __glibc if present - self._platforms = [] - if platform_virtual_packages: - # Get the glibc_version from virtual packages, falling back to - # the last of MANYLINUX_TAGS when absent - glibc_version = MANYLINUX_TAGS[-1] - for p in platform_virtual_packages.values(): - if p["name"] == "__glibc": - glibc_version = p["version"] - glibc_version_splits = list(map(int, glibc_version.split("."))) - for tag in MANYLINUX_TAGS: - if tag[0] == "_": - # Compare to see if glibc_version is greater than this version - if list(map(int, tag[1:].split("_"))) > glibc_version_splits: - break - self._platforms.append(f"manylinux{tag}_{arch}") - if tag == glibc_version: # Catches 1 and 2010 case - # We go no further than the maximum specific GLIBC version - break - # Latest tag is most preferred so list first - self._platforms.reverse() - else: - self._platforms = [ - f"manylinux{tag}_{arch}" for tag in reversed(MANYLINUX_TAGS) - ] - self._platforms.append(f"linux_{arch}") + compatible_manylinux_tags = _compute_compatible_manylinux_tags( + platform_virtual_packages=platform_virtual_packages + ) + self._platforms = [ + f"manylinux{tag}_{arch}" for tag in compatible_manylinux_tags + ] + [f"linux_{arch}"] elif system == "osx": self._platforms = list(mac_platforms(MACOS_VERSION, arch)) elif platform == "win-64": @@ -140,6 +126,115 @@ def get_marker_env(self) -> Dict[str, str]: } +def _extract_glibc_version_from_virtual_packages( + platform_virtual_packages: Dict[str, dict] +) -> Optional[Version]: + """Get the glibc version from the "package" repodata of a chosen platform. + + Note that the glibc version coming from a virtual package is never a legacy + manylinux tag (i.e. 1, 2010, or 2014). Those tags predate PEP 600 which + introduced manylinux tags containing the glibc version. Currently, all + relevant glibc versions look like 2.XX. + + >>> platform_virtual_packages = { + ... "__glibc-2.17-0.tar.bz2": { + ... "name": "__glibc", + ... "version": "2.17", + ... }, + ... } + >>> _extract_glibc_version_from_virtual_packages(platform_virtual_packages) + + >>> _extract_glibc_version_from_virtual_packages({}) is None + True + """ + matches: List[Version] = [] + for p in platform_virtual_packages.values(): + if p["name"] == "__glibc": + matches.append(Version(p["version"])) + if len(matches) == 0: + return None + elif len(matches) == 1: + return matches[0] + else: + lowest = min(matches) + warnings.warn( + f"Multiple __glibc virtual package entries found! " + f"{matches=} Using the lowest version {lowest}." + ) + return lowest + + +def _glibc_version_from_manylinux_tag(tag: str) -> Version: + """ + Return the glibc version for the given manylinux tag + + >>> _glibc_version_from_manylinux_tag("2010") + + >>> _glibc_version_from_manylinux_tag("_2_28") + + """ + SPECIAL_CASES = { + "1": Version("2.5"), + "2010": Version("2.12"), + "2014": Version("2.17"), + } + if tag in SPECIAL_CASES: + return SPECIAL_CASES[tag] + elif tag.startswith("_"): + return Version(tag[1:].replace("_", ".")) + else: + raise ValueError(f"Unknown manylinux tag {tag}") + + +def _compute_compatible_manylinux_tags( + platform_virtual_packages: Optional[Dict[str, dict]] +) -> List[str]: + """Determine the manylinux tags that are compatible with the given platform. + + If there is no glibc virtual package, then assume that all manylinux tags are + compatible. + + The result is sorted in descending order in order to favor the latest. + + >>> platform_virtual_packages = { + ... "__glibc-2.24-0.tar.bz2": { + ... "name": "__glibc", + ... "version": "2.24", + ... }, + ... } + >>> _compute_compatible_manylinux_tags({}) == list(reversed(MANYLINUX_TAGS)) + True + >>> _compute_compatible_manylinux_tags(platform_virtual_packages) + ['_2_24', '_2_17', '2014', '2010', '1'] + """ + # We use MANYLINUX_TAGS but only go up to the latest supported version + # as provided by __glibc if present + + latest_supported_glibc_version: Optional[Version] = None + # Try to get the glibc version from the virtual packages if it exists + if platform_virtual_packages: + latest_supported_glibc_version = _extract_glibc_version_from_virtual_packages( + platform_virtual_packages + ) + # Fall back to the latest of MANYLINUX_TAGS + if latest_supported_glibc_version is None: + latest_supported_glibc_version = _glibc_version_from_manylinux_tag( + MANYLINUX_TAGS[-1] + ) + + # The glibc versions are backwards compatible, so filter the MANYLINUX_TAGS + # to those compatible with less than or equal to the latest supported + # glibc version. + # Note that MANYLINUX_TAGS is sorted in ascending order. The latest tag + # is most preferred so we reverse the order. + compatible_manylinux_tags = [ + tag + for tag in reversed(MANYLINUX_TAGS) + if _glibc_version_from_manylinux_tag(tag) <= latest_supported_glibc_version + ] + return compatible_manylinux_tags + + REQUIREMENT_PATTERN = re.compile( r""" ^ diff --git a/pyproject.toml b/pyproject.toml index b6523232..a01e81b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,7 @@ exclude = [ ] [tool.pytest.ini_options] -addopts = "-vrsx -n auto" +addopts = "--doctest-modules -vrsx -n auto" flake8-max-line-length = 105 flake8-ignore = ["docs/* ALL", "conda_lock/_version.py ALL"] filterwarnings = "ignore::DeprecationWarning" diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 94012e7c..e6997a6b 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -66,7 +66,13 @@ from conda_lock.models.channel import Channel from conda_lock.models.lock_spec import Dependency, VCSDependency, VersionedDependency from conda_lock.models.pip_repository import PipRepository -from conda_lock.pypi_solver import _strip_auth, parse_pip_requirement, solve_pypi +from conda_lock.pypi_solver import ( + MANYLINUX_TAGS, + PlatformEnv, + _strip_auth, + parse_pip_requirement, + solve_pypi, +) from conda_lock.src_parser import ( DEFAULT_PLATFORMS, LockSpecification, @@ -2508,6 +2514,78 @@ def test_pip_respects_glibc_version( assert manylinux_version == [2, 17] +def test_platformenv_linux_platforms(): + """Check that PlatformEnv correctly handles Linux platforms for wheels""" + # This is the default and maximal list of platforms that we expect + all_expected_platforms = [ + f"manylinux{glibc_ver}_x86_64" for glibc_ver in reversed(MANYLINUX_TAGS) + ] + ["linux_x86_64"] + + # Check that we get the default platforms when no virtual packages are specified + e = PlatformEnv("3.12", "linux-64") + assert e._platforms == all_expected_platforms + + # Check that we get the default platforms when the virtual packages are empty + e = PlatformEnv("3.12", "linux-64", platform_virtual_packages={}) + assert e._platforms == all_expected_platforms + + # Check that we get the default platforms when the virtual packages are nonempty + # but don't include __glibc + platform_virtual_packages = {"x.bz2": {"name": "not_glibc"}} + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == all_expected_platforms + + # Check that we get the expected platforms when using the default repodata. + # (This should include the glibc corresponding to the latest manylinux tag.) + default_repodata = default_virtual_package_repodata() + platform_virtual_packages = default_repodata.all_repodata["linux-64"]["packages"] + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == all_expected_platforms + + # Check that we get the expected platforms after removing glibc from the + # default repodata. + platform_virtual_packages = { + filename: record + for filename, record in platform_virtual_packages.items() + if record["name"] != "__glibc" + } + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == all_expected_platforms + + # Check that we get a restricted list of platforms when specifying a + # lower glibc version. + restricted_platforms = [ + "manylinux_2_17_x86_64", + "manylinux2014_x86_64", + "manylinux2010_x86_64", + "manylinux1_x86_64", + "linux_x86_64", + ] + platform_virtual_packages["__glibc-2.17-0.tar.bz2"] = dict( + name="__glibc", version="2.17" + ) + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == restricted_platforms + + # Check that a warning is raised when there are multiple glibc versions + platform_virtual_packages["__glibc-2.28-0.tar.bz2"] = dict( + name="__glibc", version="2.28" + ) + with pytest.warns(UserWarning): + e = PlatformEnv( + "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages + ) + assert e._platforms == restricted_platforms + + def test_parse_environment_file_with_pip_and_platform_selector(): """See https://github.com/conda/conda-lock/pull/564 for the context.""" env_file = TEST_DIR / "test-pip-with-platform-selector" / "environment.yml"