Skip to content

Commit

Permalink
ENH: delegate computing wheel tags to the packaging module
Browse files Browse the repository at this point in the history 
Use the wheel contents only to determine whether the wheel contains
python ABI dependent modules or other platform dependent code.

Fixes mesonbuild#142, fixes mesonbuild#189, fixes mesonbuild#190.
  • Loading branch information
@dnicolodi
dnicolodi committed Nov 10, 2022
1 parent e8afceb commit b71fc8d
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 406 deletions.
1 change: 0 additions & 1 deletion meson.build
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ py.install_sources(
'mesonpy/__init__.py',
'mesonpy/_compat.py',
'mesonpy/_elf.py',
'mesonpy/_tags.py',
'mesonpy/_util.py',
subdir: 'mesonpy',
)
257 changes: 89 additions & 168 deletions mesonpy/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import collections
import contextlib
import functools
import importlib.machinery
import io
import itertools
import json
Expand All @@ -35,6 +36,8 @@
Union
)

import packaging.tags


if sys.version_info < (3, 11):
import tomli as tomllib
Expand All @@ -43,10 +46,9 @@

import mesonpy._compat
import mesonpy._elf
import mesonpy._tags
import mesonpy._util

from mesonpy._compat import Collection, Iterator, Mapping, Path
from mesonpy._compat import Iterator, Path


if typing.TYPE_CHECKING: # pragma: no cover
Expand Down Expand Up @@ -102,9 +104,31 @@ def _init_colors() -> Dict[str, str]:
_STYLES = _init_colors() # holds the color values, should be _COLORS or _NO_COLORS


_LINUX_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.so$')
_WINDOWS_NATIVE_MODULE_REGEX = re.compile(r'^(?P<name>.+)\.(?P<tag>.+)\.pyd$')
_STABLE_ABI_TAG_REGEX = re.compile(r'^abi(?P<abi_number>[0-9]+)$')
_EXTENSION_SUFFIXES = frozenset(s.lstrip('.') for s in importlib.machinery.EXTENSION_SUFFIXES)
_EXTENSION_SUFFIX_REGEX = re.compile(r'^(?:(?P<abi>[^.]+)\.)?(?:so|pyd)$')


def _adjust_manylinux_tag(platform: str) -> str:
# The packaging module generates overly specific platforms tags on
# Linux. The platforms tags on Linux evolved over time. Relax
# the platform tags to maintain compatibility with old wheel
# installation tools. The relaxed platform tags match the ones
# generated by the wheel package.
# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/
return re.sub(r'^manylinux(1|2010|2014|_\d+_\d+)_(.*)$', r'linux_\2', platform)


def _adjust_darwin_tag(platform: str) -> str:
# Override the macOS version if one is provided via the
# MACOS_DEPLOYMENT_TARGET environment variable. Return it
# unchanged otherwise.
try:
version = tuple(map(int, os.environ.get('MACOS_DEPLOYMENT_TARGET').split('.')))[:2]
except ValueError:
version = None
if version is not None:
return next(packaging.tags.mac_platforms(version))
return platform


def _showwarning(
Expand Down Expand Up @@ -179,6 +203,11 @@ def _wheel_files(self) -> DefaultDict[str, List[Tuple[pathlib.Path, str]]]:
def _has_internal_libs(self) -> bool:
return bool(self._wheel_files['mesonpy-libs'])

@property
def _has_extension_modules(self) -> bool:
# Assume that all code installed in {platlib} is Python ABI dependent.
return bool(self._wheel_files['platlib'])

@property
def basename(self) -> str:
"""Normalized wheel name and version (eg. meson_python-1.0.0)."""
Expand All @@ -187,14 +216,34 @@ def basename(self) -> str:
version=self._project.version,
)

@property
def tag(self) -> packaging.tags.Tag:
"""Wheel tags."""
if self.is_pure:
return packaging.tags.Tag('py3', 'none', 'any')
# Get the most specific tag for the Python interpreter.
tag = next(packaging.tags.sys_tags())
if tag.platform.startswith('manylinux'):
tag = packaging.tags.Tag(tag.interpreter, tag.abi, _adjust_manylinux_tag(tag.platform))
elif tag.platform.startswith('darwin'):
tag = packaging.tags.Tag(tag.interpreter, tag.abi, _adjust_darwin_tag(tag.platform))
if not self._has_extension_modules:
# The wheel has platform dependent code (is not pure) but
# does not contain any extension module (does not
# distribute any file in {platlib}) thus use generic
# implementation and ABI tags.
return packaging.tags.Tag('py3', 'none', tag.platform)
if self._stable_abi:
# All distributed extension modules use the stable ABI.
return packaging.tags.Tag(tag.interpreter, self._stable_abi, tag.platform)
return tag

@property
def name(self) -> str:
"""Wheel name, this includes the basename and tags."""
return '{basename}-{python_tag}-{abi_tag}-{platform_tag}'.format(
"""Wheel name, this includes the basename and tag."""
return '{basename}-{tag}'.format(
basename=self.basename,
python_tag=self.python_tag,
abi_tag=self.abi_tag,
platform_tag=self.platform_tag,
tag=self.tag,
)

@property
Expand Down Expand Up @@ -226,10 +275,10 @@ def wheel(self) -> bytes: # noqa: F811
Wheel-Version: 1.0
Generator: meson
Root-Is-Purelib: {is_purelib}
Tag: {tags}
Tag: {tag}
''').strip().format(
is_purelib='true' if self.is_pure else 'false',
tags=f'{self.python_tag}-{self.abi_tag}-{self.platform_tag}',
tag=self.tag,
).encode()

@property
Expand Down Expand Up @@ -267,166 +316,38 @@ def _debian_python(self) -> bool:
except ModuleNotFoundError:
return False

@property
def python_tag(self) -> str:
selected_tag = self._select_abi_tag()
if selected_tag and selected_tag.python:
return selected_tag.python
return 'py3'

@property
def abi_tag(self) -> str:
selected_tag = self._select_abi_tag()
if selected_tag:
return selected_tag.abi
return 'none'

@cached_property
def platform_tag(self) -> str:
if self.is_pure:
return 'any'
# XXX: Choose the sysconfig platform here and let something like auditwheel
# fix it later if there are system dependencies (eg. replace it with a manylinux tag)
platform_ = sysconfig.get_platform()
parts = platform_.split('-')
if parts[0] == 'macosx':
target = os.environ.get('MACOSX_DEPLOYMENT_TARGET')
if target:
print(
'{yellow}MACOSX_DEPLOYMENT_TARGET is set so we are setting the '
'platform tag to {target}{reset}'.format(target=target, **_STYLES)
)
parts[1] = target
else:
# If no target macOS version is specified fallback to
# platform.mac_ver() instead of sysconfig.get_platform() as the
# latter specifies the target macOS version Python was built
# against.
parts[1] = platform.mac_ver()[0]
if parts[1] >= '11':
# Only pick up the major version, which changed from 10.X
# to X.0 from macOS 11 onwards. See
# https://github.com/mesonbuild/meson-python/issues/160
parts[1] = parts[1].split('.')[0]

if parts[1] in ('11', '12'):
# Workaround for bug where pypa/packaging does not consider macOS
# tags without minor versions valid. Some Python flavors (Homebrew
# for example) on macOS started to do this in version 11, and
# pypa/packaging should handle things correctly from version 13 and
# forward, so we will add a 0 minor version to MacOS 11 and 12.
# https://github.com/mesonbuild/meson-python/issues/91
# https://github.com/pypa/packaging/issues/578
parts[1] += '.0'

platform_ = '-'.join(parts)
elif parts[0] == 'linux' and parts[1] == 'x86_64' and sys.maxsize == 0x7fffffff:
# 32-bit Python running on an x86_64 host
# https://github.com/mesonbuild/meson-python/issues/123
parts[1] = 'i686'
platform_ = '-'.join(parts)
return platform_.replace('-', '_').replace('.', '_')

def _calculate_file_abi_tag_heuristic_windows(self, filename: str) -> Optional[mesonpy._tags.Tag]:
"""Try to calculate the Windows tag from the Python extension file name."""
match = _WINDOWS_NATIVE_MODULE_REGEX.match(filename)
if not match:
return None
tag = match.group('tag')
def _stable_abi(self) -> Optional[str]:
"""Determine stabe ABI compatibility.
try:
return mesonpy._tags.StableABITag(tag)
except ValueError:
return mesonpy._tags.InterpreterTag(tag)

def _calculate_file_abi_tag_heuristic_posix(self, filename: str) -> Optional[mesonpy._tags.Tag]:
"""Try to calculate the Posix tag from the Python extension file name."""
# sysconfig is not guaranted to export SHLIB_SUFFIX but let's be
# preventive and check its value to make sure it matches our expectations
try:
extension = sysconfig.get_config_vars().get('SHLIB_SUFFIX', '.so')
if extension != '.so':
raise NotImplementedError(
f"We don't currently support the {extension} extension. "
'Please report this to https://github.com/mesonbuild/mesonpy/issues '
'and include information about your operating system.'
)
except KeyError:
warnings.warn(
'sysconfig does not export SHLIB_SUFFIX, so we are unable to '
'perform the sanity check regarding the extension suffix. '
'Please report this to https://github.com/mesonbuild/mesonpy/issues '
'and include the output of `python -m sysconfig`.'
)
match = _LINUX_NATIVE_MODULE_REGEX.match(filename)
if not match: # this file does not appear to be a native module
return None
tag = match.group('tag')
Examine all files installed in {platlib} that look like
extension modules (extension .pyd on Windows and .so on other
platforms) and, if they all share the same PEP 3149 filename
stable ABI tag, return it.
try:
return mesonpy._tags.StableABITag(tag)
except ValueError:
return mesonpy._tags.InterpreterTag(tag)

def _calculate_file_abi_tag_heuristic(self, filename: str) -> Optional[mesonpy._tags.Tag]:
"""Try to calculate the ABI tag from the Python extension file name."""
if os.name == 'nt':
return self._calculate_file_abi_tag_heuristic_windows(filename)
# everything else *should* follow the POSIX way, at least to my knowledge
return self._calculate_file_abi_tag_heuristic_posix(filename)

def _file_list_repr(self, files: Collection[str], prefix: str = '\t\t', max_count: int = 3) -> str:
if len(files) > max_count:
files = list(itertools.islice(files, max_count)) + [f'(... +{len(files)}))']
return ''.join(f'{prefix}- {file}\n' for file in files)

def _files_by_tag(self) -> Mapping[mesonpy._tags.Tag, Collection[str]]:
"""Map files into ABI tags."""
files_by_tag: Dict[mesonpy._tags.Tag, List[str]] = collections.defaultdict(list)

for _, file in self._wheel_files['platlib']:
# if in platlib, calculate the ABI tag
tag = self._calculate_file_abi_tag_heuristic(file)
if tag:
files_by_tag[tag].append(file)

return files_by_tag

def _select_abi_tag(self) -> Optional[mesonpy._tags.Tag]: # noqa: C901
"""Given a list of ABI tags, selects the most specific one.
Raises an error if there are incompatible tags.
All files that look like extension modules are verified to
have a file name compatibel with what is expected by the
Python interpreter. An exception is raised otherwise.
Other files are ignored.
"""
# Possibilities:
# - interpreter specific (cpython/pypy/etc, version)
# - stable abi (abiX)
tags = self._files_by_tag()
selected_tag = None
for tag, files in tags.items():
# no selected tag yet, let's assign this one
if not selected_tag:
selected_tag = tag
# interpreter tag
elif isinstance(tag, mesonpy._tags.InterpreterTag):
if tag != selected_tag:
if isinstance(selected_tag, mesonpy._tags.InterpreterTag):
raise ValueError(
'Found files with incompatible ABI tags:\n'
+ self._file_list_repr(tags[selected_tag])
+ '\tand\n'
+ self._file_list_repr(files)
)
selected_tag = tag
# stable ABI
elif isinstance(tag, mesonpy._tags.StableABITag):
if isinstance(selected_tag, mesonpy._tags.StableABITag) and tag != selected_tag:
raise ValueError(
'Found files with incompatible ABI tags:\n'
+ self._file_list_repr(tags[selected_tag])
+ '\tand\n'
+ self._file_list_repr(files)
)
return selected_tag
abis = []

for path, src in self._wheel_files['platlib']:
if os.name == 'nt' and path.suffix == '.pyd' or path.suffix == '.so':
name, suffix = path.name.split('.', 1)
if suffix not in _EXTENSION_SUFFIXES:
raise ValueError('Extension module "{}" not compatible with Python interpreter.'.format(path))
match = _EXTENSION_SUFFIX_REGEX.match(suffix)
if match is None:
raise ValueError('Extension module "{}" suffix not understood.'.format(path))
abis.append(match.group('abi') or '')

stable = [x for x in abis if re.match(r'abi\d+', x)]
if len(stable) > 0 and len(stable) == len(abis) and all(x == stable[0] for x in stable[1:]):
return stable[0]
return None


def _is_native(self, file: Union[str, pathlib.Path]) -> bool:
"""Check if file is a native file."""
Expand Down
7 changes: 2 additions & 5 deletions mesonpy/_compat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,9 @@


if sys.version_info >= (3, 9):
from collections.abc import (
Collection, Iterable, Iterator, Mapping, Sequence
)
from collections.abc import Collection, Iterable, Iterator, Sequence
else:
from typing import Collection, Iterable, Iterator, Mapping, Sequence
from typing import Collection, Iterable, Iterator, Sequence


if sys.version_info >= (3, 8):
Expand All @@ -41,7 +39,6 @@ def is_relative_to(path: pathlib.Path, other: Union[pathlib.Path, str]) -> bool:
'Iterable',
'Iterator',
'Literal',
'Mapping',
'Path',
'Sequence',
]
Loading

0 comments on commit b71fc8d

Please sign in to comment.