Refactor manylinux tag logic (#1)

* Refactor manylinux tag logic

* Run doctests

* Remove unused logger

* Clarify that legacy manylinux tags aren't glibc versions

conda_lock/pypi_solver.py

@@ -1,14 +1,16 @@
 import re
 import sys
+import warnings
 
 from pathlib import Path
 from posixpath import expandvars
-from typing import TYPE_CHECKING, Dict, List, Optional
-from urllib.parse import urldefrag, urlparse, urlsplit, urlunsplit
+from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple
+from urllib.parse import urldefrag, urlsplit, urlunsplit
 
 from clikit.api.io.flags import VERY_VERBOSE
 from clikit.io import ConsoleIO, NullIO
 from packaging.tags import compatible_tags, cpython_tags, mac_platforms
+from packaging.version import Version
 
 from conda_lock.interfaces.vendored_poetry import (
     Chooser,
@@ -39,7 +41,6 @@
 if TYPE_CHECKING:
     from packaging.tags import Tag
 
-
 # NB: in principle these depend on the glibc on the machine creating the conda env.
 # We use tags supported by manylinux Docker images, which are likely the most common
 # in practice, see https://github.com/pypa/manylinux/blob/main/README.rst#docker-images.
@@ -55,6 +56,12 @@ class PlatformEnv(Env):
     Fake poetry Env to match PyPI distributions to the target conda environment
     """
 
+    _sys_platform: Literal["darwin", "linux", "win32"]
+    _platform_system: Literal["Darwin", "Linux", "Windows"]
+    _os_name: Literal["posix", "nt"]
+    _platforms: List[str]
+    _python_version: Tuple[int, ...]
+
     def __init__(
         self,
         python_version: str,
@@ -67,33 +74,12 @@ def __init__(
             arch = "x86_64"
 
         if system == "linux":
-            # We use MANYLINUX_TAGS but only go up to the latest supported version
-            # as provided by __glibc if present
-            self._platforms = []
-            if platform_virtual_packages:
-                # Get the glibc_version from virtual packages, falling back to
-                # the last of MANYLINUX_TAGS when absent
-                glibc_version = MANYLINUX_TAGS[-1]
-                for p in platform_virtual_packages.values():
-                    if p["name"] == "__glibc":
-                        glibc_version = p["version"]
-                glibc_version_splits = list(map(int, glibc_version.split(".")))
-                for tag in MANYLINUX_TAGS:
-                    if tag[0] == "_":
-                        # Compare to see if glibc_version is greater than this version
-                        if list(map(int, tag[1:].split("_"))) > glibc_version_splits:
-                            break
-                    self._platforms.append(f"manylinux{tag}_{arch}")
-                    if tag == glibc_version:  # Catches 1 and 2010 case
-                        # We go no further than the maximum specific GLIBC version
-                        break
-                # Latest tag is most preferred so list first
-                self._platforms.reverse()
-            else:
-                self._platforms = [
-                    f"manylinux{tag}_{arch}" for tag in reversed(MANYLINUX_TAGS)
-                ]
-            self._platforms.append(f"linux_{arch}")
+            compatible_manylinux_tags = _compute_compatible_manylinux_tags(
+                platform_virtual_packages=platform_virtual_packages
+            )
+            self._platforms = [
+                f"manylinux{tag}_{arch}" for tag in compatible_manylinux_tags
+            ] + [f"linux_{arch}"]
         elif system == "osx":
             self._platforms = list(mac_platforms(MACOS_VERSION, arch))
         elif platform == "win-64":
@@ -140,6 +126,115 @@ def get_marker_env(self) -> Dict[str, str]:
         }
 
 
+def _extract_glibc_version_from_virtual_packages(
+    platform_virtual_packages: Dict[str, dict]
+) -> Optional[Version]:
+    """Get the glibc version from the "package" repodata of a chosen platform.
+
+    Note that the glibc version coming from a virtual package is never a legacy
+    manylinux tag (i.e. 1, 2010, or 2014). Those tags predate PEP 600 which
+    introduced manylinux tags containing the glibc version. Currently, all
+    relevant glibc versions look like 2.XX.
+
+    >>> platform_virtual_packages = {
+    ...     "__glibc-2.17-0.tar.bz2": {
+    ...         "name": "__glibc",
+    ...         "version": "2.17",
+    ...     },
+    ... }
+    >>> _extract_glibc_version_from_virtual_packages(platform_virtual_packages)
+    <Version('2.17')>
+    >>> _extract_glibc_version_from_virtual_packages({}) is None
+    True
+    """
+    matches: List[Version] = []
+    for p in platform_virtual_packages.values():
+        if p["name"] == "__glibc":
+            matches.append(Version(p["version"]))
+    if len(matches) == 0:
+        return None
+    elif len(matches) == 1:
+        return matches[0]
+    else:
+        lowest = min(matches)
+        warnings.warn(
+            f"Multiple __glibc virtual package entries found! "
+            f"{matches=} Using the lowest version {lowest}."
+        )
+        return lowest
+
+
+def _glibc_version_from_manylinux_tag(tag: str) -> Version:
+    """
+    Return the glibc version for the given manylinux tag
+
+    >>> _glibc_version_from_manylinux_tag("2010")
+    <Version('2.12')>
+    >>> _glibc_version_from_manylinux_tag("_2_28")
+    <Version('2.28')>
+    """
+    SPECIAL_CASES = {
+        "1": Version("2.5"),
+        "2010": Version("2.12"),
+        "2014": Version("2.17"),
+    }
+    if tag in SPECIAL_CASES:
+        return SPECIAL_CASES[tag]
+    elif tag.startswith("_"):
+        return Version(tag[1:].replace("_", "."))
+    else:
+        raise ValueError(f"Unknown manylinux tag {tag}")
+
+
+def _compute_compatible_manylinux_tags(
+    platform_virtual_packages: Optional[Dict[str, dict]]
+) -> List[str]:
+    """Determine the manylinux tags that are compatible with the given platform.
+
+    If there is no glibc virtual package, then assume that all manylinux tags are
+    compatible.
+
+    The result is sorted in descending order in order to favor the latest.
+
+    >>> platform_virtual_packages = {
+    ...     "__glibc-2.24-0.tar.bz2": {
+    ...         "name": "__glibc",
+    ...         "version": "2.24",
+    ...     },
+    ... }
+    >>> _compute_compatible_manylinux_tags({}) == list(reversed(MANYLINUX_TAGS))
+    True
+    >>> _compute_compatible_manylinux_tags(platform_virtual_packages)
+    ['_2_24', '_2_17', '2014', '2010', '1']
+    """
+    # We use MANYLINUX_TAGS but only go up to the latest supported version
+    # as provided by __glibc if present
+
+    latest_supported_glibc_version: Optional[Version] = None
+    # Try to get the glibc version from the virtual packages if it exists
+    if platform_virtual_packages:
+        latest_supported_glibc_version = _extract_glibc_version_from_virtual_packages(
+            platform_virtual_packages
+        )
+    # Fall back to the latest of MANYLINUX_TAGS
+    if latest_supported_glibc_version is None:
+        latest_supported_glibc_version = _glibc_version_from_manylinux_tag(
+            MANYLINUX_TAGS[-1]
+        )
+
+    # The glibc versions are backwards compatible, so filter the MANYLINUX_TAGS
+    # to those compatible with less than or equal to the latest supported
+    # glibc version.
+    # Note that MANYLINUX_TAGS is sorted in ascending order. The latest tag
+    # is most preferred so we reverse the order.
+    compatible_manylinux_tags = [
+        tag
+        for tag in reversed(MANYLINUX_TAGS)
+        if _glibc_version_from_manylinux_tag(tag) <= latest_supported_glibc_version
+    ]
+    return compatible_manylinux_tags
+
+
 REQUIREMENT_PATTERN = re.compile(
     r"""
     ^

pyproject.toml

@@ -109,7 +109,7 @@ exclude = [
 ]
 
 [tool.pytest.ini_options]
-addopts = "-vrsx -n auto"
+addopts = "--doctest-modules -vrsx -n auto"
 flake8-max-line-length = 105
 flake8-ignore = ["docs/* ALL", "conda_lock/_version.py ALL"]
 filterwarnings = "ignore::DeprecationWarning"

tests/test_conda_lock.py

@@ -66,7 +66,13 @@
 from conda_lock.models.channel import Channel
 from conda_lock.models.lock_spec import Dependency, VCSDependency, VersionedDependency
 from conda_lock.models.pip_repository import PipRepository
-from conda_lock.pypi_solver import _strip_auth, parse_pip_requirement, solve_pypi
+from conda_lock.pypi_solver import (
+    MANYLINUX_TAGS,
+    PlatformEnv,
+    _strip_auth,
+    parse_pip_requirement,
+    solve_pypi,
+)
 from conda_lock.src_parser import (
     DEFAULT_PLATFORMS,
     LockSpecification,
@@ -2508,6 +2514,78 @@ def test_pip_respects_glibc_version(
     assert manylinux_version == [2, 17]
 
 
+def test_platformenv_linux_platforms():
+    """Check that PlatformEnv correctly handles Linux platforms for wheels"""
+    # This is the default and maximal list of platforms that we expect
+    all_expected_platforms = [
+        f"manylinux{glibc_ver}_x86_64" for glibc_ver in reversed(MANYLINUX_TAGS)
+    ] + ["linux_x86_64"]
+
+    # Check that we get the default platforms when no virtual packages are specified
+    e = PlatformEnv("3.12", "linux-64")
+    assert e._platforms == all_expected_platforms
+
+    # Check that we get the default platforms when the virtual packages are empty
+    e = PlatformEnv("3.12", "linux-64", platform_virtual_packages={})
+    assert e._platforms == all_expected_platforms
+
+    # Check that we get the default platforms when the virtual packages are nonempty
+    # but don't include __glibc
+    platform_virtual_packages = {"x.bz2": {"name": "not_glibc"}}
+    e = PlatformEnv(
+        "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages
+    )
+    assert e._platforms == all_expected_platforms
+
+    # Check that we get the expected platforms when using the default repodata.
+    # (This should include the glibc corresponding to the latest manylinux tag.)
+    default_repodata = default_virtual_package_repodata()
+    platform_virtual_packages = default_repodata.all_repodata["linux-64"]["packages"]
+    e = PlatformEnv(
+        "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages
+    )
+    assert e._platforms == all_expected_platforms
+
+    # Check that we get the expected platforms after removing glibc from the
+    # default repodata.
+    platform_virtual_packages = {
+        filename: record
+        for filename, record in platform_virtual_packages.items()
+        if record["name"] != "__glibc"
+    }
+    e = PlatformEnv(
+        "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages
+    )
+    assert e._platforms == all_expected_platforms
+
+    # Check that we get a restricted list of platforms when specifying a
+    # lower glibc version.
+    restricted_platforms = [
+        "manylinux_2_17_x86_64",
+        "manylinux2014_x86_64",
+        "manylinux2010_x86_64",
+        "manylinux1_x86_64",
+        "linux_x86_64",
+    ]
+    platform_virtual_packages["__glibc-2.17-0.tar.bz2"] = dict(
+        name="__glibc", version="2.17"
+    )
+    e = PlatformEnv(
+        "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages
+    )
+    assert e._platforms == restricted_platforms
+
+    # Check that a warning is raised when there are multiple glibc versions
+    platform_virtual_packages["__glibc-2.28-0.tar.bz2"] = dict(
+        name="__glibc", version="2.28"
+    )
+    with pytest.warns(UserWarning):
+        e = PlatformEnv(
+            "3.12", "linux-64", platform_virtual_packages=platform_virtual_packages
+        )
+    assert e._platforms == restricted_platforms
+
+
 def test_parse_environment_file_with_pip_and_platform_selector():
     """See https://github.com/conda/conda-lock/pull/564 for the context."""
     env_file = TEST_DIR / "test-pip-with-platform-selector" / "environment.yml"