From 541d090e888da4fa61d0bb223ff0b42f5d508c9d Mon Sep 17 00:00:00 2001 From: Daniele Nicolodi Date: Sun, 30 Oct 2022 20:56:17 +0100 Subject: [PATCH] Delegate computing wheel tags to the packaging module Use the wheel contents only to determine whether the wheel contains python ABI dependent modules or other platform dependent code. Fixes #189, #190. --- mesonpy/__init__.py | 203 +++++++++----------------------------------- mesonpy/_compat.py | 7 +- mesonpy/_tags.py | 134 ----------------------------- tests/test_wheel.py | 35 ++------ 4 files changed, 45 insertions(+), 334 deletions(-) delete mode 100644 mesonpy/_tags.py diff --git a/mesonpy/__init__.py b/mesonpy/__init__.py index 6a271d781..f5e969811 100644 --- a/mesonpy/__init__.py +++ b/mesonpy/__init__.py @@ -13,6 +13,7 @@ import collections import contextlib import functools +import importlib.machinery import io import itertools import json @@ -35,6 +36,8 @@ Union ) +import packaging.tags + if sys.version_info < (3, 11): import tomli as tomllib @@ -43,10 +46,9 @@ import mesonpy._compat import mesonpy._elf -import mesonpy._tags import mesonpy._util -from mesonpy._compat import Collection, Iterator, Mapping, Path +from mesonpy._compat import Collection, Iterator, Path if typing.TYPE_CHECKING: # pragma: no cover @@ -102,9 +104,17 @@ def _init_colors() -> Dict[str, str]: _STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS -_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P.+)\.(?P.+)\.so$') -_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P.+)\.(?P.+)\.pyd$') -_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P[0-9]+)$') +_EXTENSION_SUFFIXES = frozenset(s.lstrip('.') for s in importlib.machinery.EXTENSION_SUFFIXES) +_EXTENSION_SUFFIX_REGEX = re.compile(r'^(?:(?P[^.]+)\.)?(?:so|pyd)$') + + +def _extension_abi_tag(path: pathlib.Path) -> str: + """Extract the PEP 3149 ABI tag from the extension file path, if any is present.""" + # The file path cannot contain a dot. + name, suffix = path.name.split('.', 1) + if suffix not in _EXTENSION_SUFFIXES: + raise ValueError('Extension module "{}" not compatible with Python interpreter.'.format(str(path))) + return _EXTENSION_SUFFIX_REGEX.match(suffix).group('abi') def _showwarning( @@ -179,6 +189,10 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]: def _has_internal_libs(self) -> bool: return bool(self._wheel_files['mesonpy-libs']) + @property + def _has_extension_modules(self) -> bool: + return bool(self._wheel_files['platlib']) + @property def basename(self) -> str: """Normalized wheel name and version (eg. meson_python-1.0.0).""" @@ -187,14 +201,24 @@ def basename(self) -> str: version=self._project.version, ) + @property + def tags(self) -> str: + """Wheel tags.""" + if self.is_pure: + return 'py3-none-any' + tags = next(packaging.tags.sys_tags()) + if not self._has_extension_modules: + tags.abi = 'none' + elif self._use_stable_abi: + tags.abi = 'abi3' + return tags + @property def name(self) -> str: """Wheel name, this includes the basename and tags.""" - return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format( + return '{basename}-{tags}'.format( basename=self.basename, - python_tag=self.python_tag, - abi_tag=self.abi_tag, - platform_tag=self.platform_tag, + tags=str(self.tags), ) @property @@ -229,7 +253,7 @@ def wheel(self) -> bytes: # noqa: F811 Tag: {tags} ''').strip().format( is_purelib='true' if self.is_pure else 'false', - tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}', + tags=self.tags, ).encode() @property @@ -267,166 +291,15 @@ def _debian_python(self) -> bool: except ModuleNotFoundError: return False - @property - def python_tag(self) -> str: - selected_tag = self._select_abi_tag() - if selected_tag and selected_tag.python: - return selected_tag.python - return 'py3' - - @property - def abi_tag(self) -> str: - selected_tag = self._select_abi_tag() - if selected_tag: - return selected_tag.abi - return 'none' - - @cached_property - def platform_tag(self) -> str: - if self.is_pure: - return 'any' - # XXX: Choose the sysconfig platform here and let something like auditwheel - # fix it later if there are system dependencies (eg. replace it with a manylinux tag) - platform_ = sysconfig.get_platform() - parts = platform_.split('-') - if parts[0] == 'macosx': - target = os.environ.get('MACOSX_DEPLOYMENT_TARGET') - if target: - print( - '{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the ' - 'platform tag to {target}{reset}'.format(target=target, **_STYLES) - ) - parts[1] = target - else: - # If no target macOS version is specified fallback to - # platform.mac_ver() instead of sysconfig.get_platform() as the - # latter specifies the target macOS version Python was built - # against. - parts[1] = platform.mac_ver()[0] - if parts[1] >= '11': - # Only pick up the major version, which changed from 10.X - # to X.0 from macOS 11 onwards. See - # https://github.com/mesonbuild/meson-python/issues/160 - parts[1] = parts[1].split('.')[0] - - if parts[1] in ('11', '12'): - # Workaround for bug where pypa/packaging does not consider macOS - # tags without minor versions valid. Some Python flavors (Homebrew - # for example) on macOS started to do this in version 11, and - # pypa/packaging should handle things correctly from version 13 and - # forward, so we will add a 0 minor version to MacOS 11 and 12. - # https://github.com/mesonbuild/meson-python/issues/91 - # https://github.com/pypa/packaging/issues/578 - parts[1] += '.0' - - platform_ = '-'.join(parts) - elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff: - # 32-bit Python running on an x86_64 host - # https://github.com/mesonbuild/meson-python/issues/123 - parts[1] = 'i686' - platform_ = '-'.join(parts) - return platform_.replace('-', '_').replace('.', '_') - - def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the Windows tag from the Python extension file name.""" - match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename) - if not match: - return None - tag = match.group('tag') - - try: - return mesonpy._tags.StableABITag(tag) - except ValueError: - return mesonpy._tags.InterpreterTag(tag) - - def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the Posix tag from the Python extension file name.""" - # sysconfig is not guaranted to export SHLIB_SUFFIX but let's be - # preventive and check its value to make sure it matches our expectations - try: - extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so') - if extension != '.so': - raise NotImplementedError( - f"We don't currently support the {extension} extension. " - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include information about your operating system.' - ) - except KeyError: - warnings.warn( - 'sysconfig does not export SHLIB_SUFFIX, so we are unable to ' - 'perform the sanity check regarding the extension suffix. ' - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include the output of `python -m sysconfig`.' - ) - match = _LINUX_NATIVE_MODULE_REGEX.match(filename) - if not match: # this file does not appear to be a native module - return None - tag = match.group('tag') - - try: - return mesonpy._tags.StableABITag(tag) - except ValueError: - return mesonpy._tags.InterpreterTag(tag) - - def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]: - """Try to calculate the ABI tag from the Python extension file name.""" - if os.name == 'nt': - return self._calculate_file_abi_tag_heuristic_windows(filename) - # everything else *should* follow the POSIX way, at least to my knowledge - return self._calculate_file_abi_tag_heuristic_posix(filename) - def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str: if len(files) > max_count: files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))'] return ''.join(f'{prefix}- {file}\n' for file in files) - def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]: - """Map files into ABI tags.""" - files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list) - - for _, file in self._wheel_files['platlib']: - # if in platlib, calculate the ABI tag - tag = self._calculate_file_abi_tag_heuristic(file) - if tag: - files_by_tag[tag].append(file) - - return files_by_tag - - def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]: # noqa: C901 - """Given a list of ABI tags, selects the most specific one. - - Raises an error if there are incompatible tags. - """ - # Possibilities: - # - interpreter specific (cpython/pypy/etc, version) - # - stable abi (abiX) - tags = self._files_by_tag() - selected_tag = None - for tag, files in tags.items(): - # no selected tag yet, let's assign this one - if not selected_tag: - selected_tag = tag - # interpreter tag - elif isinstance(tag, mesonpy._tags.InterpreterTag): - if tag != selected_tag: - if isinstance(selected_tag, mesonpy._tags.InterpreterTag): - raise ValueError( - 'Found files with incompatible ABI tags:\n' - + self._file_list_repr(tags[selected_tag]) - + '\tand\n' - + self._file_list_repr(files) - ) - selected_tag = tag - # stable ABI - elif isinstance(tag, mesonpy._tags.StableABITag): - if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag: - raise ValueError( - 'Found files with incompatible ABI tags:\n' - + self._file_list_repr(tags[selected_tag]) - + '\tand\n' - + self._file_list_repr(files) - ) - return selected_tag + @cached_property + def _use_stable_abi(self) -> bool: + """Determine wether the package is compatible with the stabe ABI.""" + return all(_extension_abi_tag(path) == 'abi3' for path, _ in self._wheel_files['platlib']) def _is_native(self, file: Union[str, pathlib.Path]) -> bool: """Check if file is a native file.""" diff --git a/mesonpy/_compat.py b/mesonpy/_compat.py index 9fab9054f..dc74de40b 100644 --- a/mesonpy/_compat.py +++ b/mesonpy/_compat.py @@ -10,11 +10,9 @@ if sys.version_info >= (3, 9): - from collections.abc import ( - Collection, Iterable, Iterator, Mapping, Sequence - ) + from collections.abc import Collection, Iterable, Iterator, Sequence else: - from typing import Collection, Iterable, Iterator, Mapping, Sequence + from typing import Collection, Iterable, Iterator, Sequence if sys.version_info >= (3, 8): @@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool: 'Iterable', 'Iterator', 'Literal', - 'Mapping', 'Path', 'Sequence', ] diff --git a/mesonpy/_tags.py b/mesonpy/_tags.py deleted file mode 100644 index d9a9b3ba8..000000000 --- a/mesonpy/_tags.py +++ /dev/null @@ -1,134 +0,0 @@ -# SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Quansight, LLC -# SPDX-FileCopyrightText: 2021 Filipe Laíns - -import abc -import re - -from typing import Any, Optional - -from mesonpy._compat import Literal, Sequence - - -class Tag(abc.ABC): - @abc.abstractmethod - def __init__(self, value: str) -> None: ... - - @abc.abstractmethod - def __str__(self) -> str: ... - - @property - @abc.abstractmethod - def python(self) -> Optional[str]: - """Python tag.""" - - @property - @abc.abstractmethod - def abi(self) -> str: - """ABI tag.""" - - -class StableABITag(Tag): - _REGEX = re.compile(r'^abi(?P[0-9]+)$') - - def __init__(self, value: str) -> None: - match = self._REGEX.match(value) - if not match: - raise ValueError(f'Invalid PEP 3149 stable ABI tag, expecting pattern `{self._REGEX.pattern}`') - self._abi_number = int(match.group('abi_number')) - - @property - def abi_number(self) -> int: - return self._abi_number - - def __str__(self) -> str: - return f'abi{self.abi_number}' - - @property - def python(self) -> Literal[None]: - return None - - @property - def abi(self) -> str: - return f'abi{self.abi_number}' - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and other.abi_number == self.abi_number - - def __hash__(self) -> int: - return hash(str(self)) - - -class InterpreterTag(Tag): - def __init__(self, value: str) -> None: - parts = value.split('-') - if len(parts) < 2: - raise ValueError( - 'Invalid PEP 3149 interpreter tag, expected at ' - f'least 2 parts but got {len(parts)}' - ) - - # On Windows, file extensions look like `.cp311-win_amd64.pyd`, so the - # implementation part (`cpython-`) is different from Linux. Handle that here: - if parts[0].startswith('cp3'): - parts.insert(0, 'cpython') - parts[1] = parts[1][2:] # strip 'cp' - - self._implementation = parts[0] - self._interpreter_version = parts[1] - self._additional_information = parts[2:] - - if self.implementation != 'cpython' and not self.implementation.startswith('pypy'): - raise NotImplementedError( - f'Unknown Python implementation: {self.implementation}. ' - 'Please report this to https://github.com/mesonbuild/mesonpy/issues ' - 'and include information about the Python distribution you are using.' - ) - - @property - def implementation(self) -> str: - return self._implementation - - @property - def interpreter_version(self) -> str: - return self._interpreter_version - - @property - def additional_information(self) -> Sequence[str]: - return tuple(self._additional_information) - - def __str__(self) -> str: - return '-'.join(( - self.implementation, - self.interpreter_version, - *self.additional_information, - )) - - @property - def python(self) -> str: - if self.implementation == 'cpython': - # The Python tag for CPython does not seem to include the flags suffixes. - return f'cp{self.interpreter_version}'.rstrip('dmu') - elif self.implementation.startswith('pypy'): - return f'pp{self.implementation[4:]}' - # XXX: FYI older PyPy version use the following model - # pp{self.implementation[4]}{self.interpreter_version[2:]} - raise ValueError(f'Unknown implementation: {self.implementation}') - - @property - def abi(self) -> str: - if self.implementation == 'cpython': - return f'cp{self.interpreter_version}' - elif self.implementation.startswith('pypy'): - return f'{self.implementation}_{self.interpreter_version}' - raise ValueError(f'Unknown implementation: {self.implementation}') - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, self.__class__) - and other.implementation == self.implementation - and other.interpreter_version == self.interpreter_version - ) - - def __hash__(self) -> int: - return hash(str(self)) diff --git a/tests/test_wheel.py b/tests/test_wheel.py index 1fed555f5..0817e1a2d 100644 --- a/tests/test_wheel.py +++ b/tests/test_wheel.py @@ -8,6 +8,7 @@ import sysconfig import textwrap +import packaging.tags import pytest import wheel.wheelfile @@ -25,35 +26,10 @@ EXT_SUFFIX = sysconfig.get_config_var('EXT_SUFFIX') INTERPRETER_VERSION = f'{sys.version_info[0]}{sys.version_info[1]}' - -if platform.python_implementation() == 'CPython': - INTERPRETER_TAG = f'cp{INTERPRETER_VERSION}' - PYTHON_TAG = INTERPRETER_TAG - # Py_UNICODE_SIZE has been a runtime option since Python 3.3, - # so the u suffix no longer exists - if sysconfig.get_config_var('Py_DEBUG'): - INTERPRETER_TAG += 'd' - # https://github.com/pypa/packaging/blob/5984e3b25f4fdee64aad20e98668c402f7ed5041/packaging/tags.py#L147-L150 - if sys.version_info < (3, 8): - pymalloc = sysconfig.get_config_var('WITH_PYMALLOC') - if pymalloc or pymalloc is None: # none is the default value, which is enable - INTERPRETER_TAG += 'm' -elif platform.python_implementation() == 'PyPy': - INTERPRETER_TAG = sysconfig.get_config_var('SOABI').replace('-', '_') - PYTHON_TAG = f'pp{INTERPRETER_VERSION}' -else: - raise NotImplementedError(f'Unknown implementation: {platform.python_implementation()}') - -platform_ = sysconfig.get_platform() -if platform.system() == 'Darwin': - parts = platform_.split('-') - parts[1] = platform.mac_ver()[0] - if parts[1] >= '11': - parts[1] = parts[1].split('.')[0] - if parts[1] in ('11', '12'): - parts[1] += '.0' - platform_ = '-'.join(parts) -PLATFORM_TAG = platform_.replace('-', '_').replace('.', '_') +_tag = next(packaging.tags.sys_tags()) +INTERPRETER_TAG = _tag.interpreter +PYTHON_TAG = _tag.abi +PLATFORM_TAG = _tag.platform if platform.system() == 'Linux': SHARED_LIB_EXT = 'so' @@ -80,7 +56,6 @@ def wheel_filename(artifact): win_py37 = os.name == 'nt' and sys.version_info < (3, 8) -@pytest.mark.skipif(win_py37, reason='An issue with missing file extension') def test_scipy_like(wheel_scipy_like): # This test is meant to exercise features commonly needed by a regular # Python package for scientific computing or data science: