From 62d84a5aae229aa13c3ca41b6c1a91706e9daccd Mon Sep 17 00:00:00 2001 From: mayeut Date: Thu, 26 Sep 2019 00:03:39 +0200 Subject: [PATCH 1/2] Add manylinux2014 support Per PEP 599: https://www.python.org/dev/peps/pep-0599/ --- news/7102.feature | 4 ++ src/pip/_internal/pep425tags.py | 32 ++++++++++++++ tests/functional/test_download.py | 4 ++ tests/unit/test_pep425tags.py | 71 ++++++++++++++++++++++++++++--- 4 files changed, 104 insertions(+), 7 deletions(-) create mode 100644 news/7102.feature diff --git a/news/7102.feature b/news/7102.feature new file mode 100644 index 00000000000..4412649fcb3 --- /dev/null +++ b/news/7102.feature @@ -0,0 +1,4 @@ +Implement manylinux2014 platform tag support. manylinux2014 is the successor +to manylinux2010. It allows carefully compiled binary wheels to be installed +on compatible Linux platforms. The manylinux2014 platform tag definition can +be found in `PEP599 `_. diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index f60d7a63707..9225b1b9bed 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -200,6 +200,26 @@ def is_manylinux2010_compatible(): return pip._internal.utils.glibc.have_compatible_glibc(2, 12) +def is_manylinux2014_compatible(): + # type: () -> bool + # Only Linux and only x86-64, i686, aarch64, armv7l, ppc64, ppc64le, s390x + if get_platform() not in {"linux_x86_64", "linux_i686", "linux_aarch64", + "linux_armv7l", "linux_ppc64", "linux_ppc64le", + "linux_s390x"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2014_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 7 uses glibc 2.17. + return pip._internal.utils.glibc.have_compatible_glibc(2, 17) + + def get_darwin_arches(major, minor, machine): # type: (int, int, str) -> List[str] """Return a list of supported arches (including group arches) for @@ -333,6 +353,16 @@ def get_supported( else: # arch pattern didn't match (?!) arches = [arch] + elif arch_prefix == 'manylinux2014': + arches = [arch] + # manylinux1/manylinux2010 wheels run on most manylinux2014 systems + # with the exception of wheels depending on ncurses. PEP 599 states + # manylinux1/manylinux2010 wheels should be considered + # manylinux2014 wheels: + # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels + if arch_suffix in {'i686', 'x86_64'}: + arches.append('manylinux2010' + arch_sep + arch_suffix) + arches.append('manylinux1' + arch_sep + arch_suffix) elif arch_prefix == 'manylinux2010': # manylinux1 wheels run on most manylinux2010 systems with the # exception of wheels depending on ncurses. PEP 571 states @@ -341,6 +371,8 @@ def get_supported( arches = [arch, 'manylinux1' + arch_sep + arch_suffix] elif platform is None: arches = [] + if is_manylinux2014_compatible(): + arches.append('manylinux2014' + arch_sep + arch_suffix) if is_manylinux2010_compatible(): arches.append('manylinux2010' + arch_sep + arch_suffix) if is_manylinux1_compatible(): diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 7873e255b61..77f54e8ae22 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -331,6 +331,7 @@ class TestDownloadPlatformManylinuxes(object): "linux_x86_64", "manylinux1_x86_64", "manylinux2010_x86_64", + "manylinux2014_x86_64", ]) def test_download_universal(self, platform, script, data): """ @@ -353,6 +354,9 @@ def test_download_universal(self, platform, script, data): ("manylinux1_x86_64", "manylinux1_x86_64"), ("manylinux1_x86_64", "manylinux2010_x86_64"), ("manylinux2010_x86_64", "manylinux2010_x86_64"), + ("manylinux1_x86_64", "manylinux2014_x86_64"), + ("manylinux2010_x86_64", "manylinux2014_x86_64"), + ("manylinux2014_x86_64", "manylinux2014_x86_64"), ]) def test_download_compatible_manylinuxes( self, wheel_abi, platform, script, data, diff --git a/tests/unit/test_pep425tags.py b/tests/unit/test_pep425tags.py index a18f525a98f..c43843043aa 100644 --- a/tests/unit/test_pep425tags.py +++ b/tests/unit/test_pep425tags.py @@ -137,6 +137,7 @@ def test_manual_abi_dm_flags(self): @pytest.mark.parametrize('is_manylinux_compatible', [ pep425tags.is_manylinux1_compatible, pep425tags.is_manylinux2010_compatible, + pep425tags.is_manylinux2014_compatible, ]) class TestManylinuxTags(object): """ @@ -156,28 +157,28 @@ def test_manylinux_compatible_on_linux_x86_64(self, @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_i686') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) - def test_manylinux1_compatible_on_linux_i686(self, - is_manylinux_compatible): + def test_manylinux_compatible_on_linux_i686(self, + is_manylinux_compatible): """ - Test that manylinux1 is enabled on linux_i686 + Test that manylinuxes are enabled on linux_i686 """ assert is_manylinux_compatible() @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: False) - def test_manylinux1_2(self, is_manylinux_compatible): + def test_manylinux_2(self, is_manylinux_compatible): """ - Test that manylinux1 is disabled with incompatible glibc + Test that manylinuxes are disabled with incompatible glibc """ assert not is_manylinux_compatible() @patch('pip._internal.pep425tags.get_platform', lambda: 'arm6vl') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) - def test_manylinux1_3(self, is_manylinux_compatible): + def test_manylinux_3(self, is_manylinux_compatible): """ - Test that manylinux1 is disabled on arm6vl + Test that manylinuxes are disabled on arm6vl """ assert not is_manylinux_compatible() @@ -186,6 +187,8 @@ class TestManylinux1Tags(object): @patch('pip._internal.pep425tags.is_manylinux2010_compatible', lambda: False) + @patch('pip._internal.pep425tags.is_manylinux2014_compatible', + lambda: False) @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) @@ -210,6 +213,8 @@ def test_manylinux1_tag_is_first(self): class TestManylinux2010Tags(object): + @patch('pip._internal.pep425tags.is_manylinux2014_compatible', + lambda: False) @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') @patch('pip._internal.utils.glibc.have_compatible_glibc', lambda major, minor: True) @@ -253,3 +258,55 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): if arches == ['any']: continue assert arches[:2] == [manylinux2010, manylinux1] + + +class TestManylinux2014Tags(object): + + @patch('pip._internal.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('pip._internal.utils.glibc.have_compatible_glibc', + lambda major, minor: True) + @patch('sys.platform', 'linux2') + def test_manylinux2014_tag_is_first(self): + """ + Test that the more specific tag manylinux2014 comes first. + """ + groups = {} + for pyimpl, abi, arch in pep425tags.get_supported(): + groups.setdefault((pyimpl, abi), []).append(arch) + + for arches in groups.values(): + if arches == ['any']: + continue + # Expect the most specific arch first: + if len(arches) == 5: + assert arches == ['manylinux2014_x86_64', + 'manylinux2010_x86_64', + 'manylinux1_x86_64', + 'linux_x86_64', + 'any'] + else: + assert arches == ['manylinux2014_x86_64', + 'manylinux2010_x86_64', + 'manylinux1_x86_64', + 'linux_x86_64'] + + @pytest.mark.parametrize("manylinuxA,manylinuxB", [ + ("manylinux2014_x86_64", ["manylinux2010_x86_64", + "manylinux1_x86_64"]), + ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), + ]) + def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): + """ + Specifying manylinux2014 implies manylinux2010/manylinux1. + """ + groups = {} + supported = pep425tags.get_supported(platform=manylinuxA) + for pyimpl, abi, arch in supported: + groups.setdefault((pyimpl, abi), []).append(arch) + + expected_arches = [manylinuxA] + expected_arches.extend(manylinuxB) + for arches in groups.values(): + if arches == ['any']: + continue + assert arches[:3] == expected_arches From d3d3ccad47f605ac1c1f35f824d67e4afa8c8c13 Mon Sep 17 00:00:00 2001 From: mayeut Date: Wed, 2 Oct 2019 22:55:19 +0200 Subject: [PATCH 2/2] Detect armv7l hard-float ABI for manylinux2014 PEP 599 defines manylinux2014 armv7l to be compatible with centos7 altarch armv7 i.e. armhf using the gnueabihf ABI --- src/pip/_internal/pep425tags.py | 40 +++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py index 9225b1b9bed..042ba34b38f 100644 --- a/src/pip/_internal/pep425tags.py +++ b/src/pip/_internal/pep425tags.py @@ -164,6 +164,32 @@ def get_platform(): return result +def is_linux_armhf(): + # type: () -> bool + if get_platform() != "linux_armv7l": + return False + # hard-float ABI can be detected from the ELF header of the running + # process + try: + with open(sys.executable, 'rb') as f: + elf_header_raw = f.read(40) # read 40 first bytes of ELF header + except (IOError, OSError, TypeError): + return False + if elf_header_raw is None or len(elf_header_raw) < 40: + return False + if isinstance(elf_header_raw, str): + elf_header = [ord(c) for c in elf_header_raw] + else: + elf_header = [b for b in elf_header_raw] + result = elf_header[0:4] == [0x7f, 0x45, 0x4c, 0x46] # ELF magic number + result &= elf_header[4:5] == [1] # 32-bit ELF + result &= elf_header[5:6] == [1] # little-endian + result &= elf_header[18:20] == [0x28, 0] # ARM machine + result &= elf_header[39:40] == [5] # ARM EABIv5 + result &= (elf_header[37:38][0] & 4) == 4 # EF_ARM_ABI_FLOAT_HARD + return result + + def is_manylinux1_compatible(): # type: () -> bool # Only Linux, and only x86-64 / i686 @@ -202,10 +228,16 @@ def is_manylinux2010_compatible(): def is_manylinux2014_compatible(): # type: () -> bool - # Only Linux and only x86-64, i686, aarch64, armv7l, ppc64, ppc64le, s390x - if get_platform() not in {"linux_x86_64", "linux_i686", "linux_aarch64", - "linux_armv7l", "linux_ppc64", "linux_ppc64le", - "linux_s390x"}: + # Only Linux, and only supported architectures + platform = get_platform() + if platform not in {"linux_x86_64", "linux_i686", "linux_aarch64", + "linux_armv7l", "linux_ppc64", "linux_ppc64le", + "linux_s390x"}: + return False + + # check for hard-float ABI in case we're running linux_armv7l not to + # install hard-float ABI wheel in a soft-float ABI environment + if platform == "linux_armv7l" and not is_linux_armhf(): return False # Check for presence of _manylinux module