From ba1ab6aaa2c44fd8d11801fe9c43718aabf3adfd Mon Sep 17 00:00:00 2001 From: Kris Wilson Date: Mon, 7 May 2018 18:09:20 -0700 Subject: [PATCH] Retrofit PR #399 changes. --- docs/api/index.rst | 14 +++ docs/buildingpex.rst | 10 +- pex/bin/pex.py | 51 +++++++--- pex/environment.py | 28 ++++-- pex/interpreter.py | 116 +++++++++++++++-------- pex/pep425tags.py | 53 ++++++----- pex/platforms.py | 80 ++++++++++++++++ pex/resolver.py | 65 ++++++++----- pex/resolver_options.py | 24 ++++- pex/testing.py | 23 +++-- tests/test_integration.py | 135 +++++++++++++++++++++++--- tests/test_interpreter.py | 9 ++ tests/test_pep425tags.py | 168 +++++++++++++++++++++++++++++++++ tests/test_pex_binary.py | 2 + tests/test_pex_bootstrapper.py | 4 +- tests/test_platform.py | 86 +++++++++++++++++ tox.ini | 2 +- 17 files changed, 725 insertions(+), 145 deletions(-) create mode 100644 pex/platforms.py create mode 100644 tests/test_pep425tags.py create mode 100644 tests/test_platform.py diff --git a/docs/api/index.rst b/docs/api/index.rst index 168e52cad..c225a0d62 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -36,6 +36,13 @@ pex.finders module :members: :show-inheritance: +pex.glibc module +---------------- + +.. automodule:: pex.glibc + :members: + :show-inheritance: + pex.http module -------------------- @@ -106,6 +113,13 @@ pex.pex_info module :members: :show-inheritance: +pex.platforms module +-------------------- + +.. automodule:: pex.platforms + :members: + :show-inheritance: + pex.resolver module ------------------- diff --git a/docs/buildingpex.rst b/docs/buildingpex.rst index 36113bc88..6325b0677 100644 --- a/docs/buildingpex.rst +++ b/docs/buildingpex.rst @@ -376,11 +376,11 @@ in certain situations when particular extensions may not be necessary to run a p The platform to build the pex for. Right now it defaults to the current system, but you can specify something like ``linux-x86_64`` or ``macosx-10.6-x86_64``. This will look for bdists for the particular platform. -To build manylinux wheels for specific tags, you can add them to the platform with hyphens like -``PLATFORM-PYVER-IMPL-ABI``, where ``PLATFORM`` is either ``manylinux1-x86_64`` or ``manylinux1-i686``, ``PYVER`` -is a two-digit string representing the python version (e.g., ``36``), ``IMPL`` is the python implementation -abbreviation (e.g., ``cp``, ``pp``, ``jp``), and ``ABI`` is the ABI tag (e.g., ``cp36m``, ``cp27mu``, ``abi3``, -``none``). A complete example: ``manylinux1_x86_64-36-cp-cp36m``. +To resolve wheels for specific interpreter/platform tags, you can append them to the platform name with hyphens +like ``PLATFORM-IMPL-PYVER-ABI``, where ``PLATFORM`` is the platform (e.g. ``linux-x86_64``, +``macosx-10.4-x86_64``), ``IMPL`` is the python implementation abbreviation (e.g. ``cp``, ``pp``, ``jp``), ``PYVER`` +is a two-digit string representing the python version (e.g., ``36``) and ``ABI`` is the ABI tag (e.g., ``cp36m``, +``cp27mu``, ``abi3``, ``none``). A complete example: ``linux_x86_64-cp-36-cp36m``. Tailoring PEX execution at runtime ---------------------------------- diff --git a/pex/bin/pex.py b/pex/bin/pex.py index db615b6a3..c0cb017c3 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -26,10 +26,10 @@ from pex.interpreter_constraints import validate_constraints from pex.iterator import Iterator from pex.package import EggPackage, SourcePackage -from pex.pep425tags import get_platform from pex.pex import PEX from pex.pex_bootstrapper import find_compatible_interpreters from pex.pex_builder import PEXBuilder +from pex.platforms import Platform from pex.requirements import requirements_from_file from pex.resolvable import Resolvable from pex.resolver import Unsatisfiable, resolve_multi @@ -124,6 +124,12 @@ def process_precedence(option, option_str, option_value, parser, builder): elif option_str in ('--no-wheel', '--no-use-wheel'): setattr(parser.values, option.dest, False) builder.no_use_wheel() + elif option_str == '--manylinux': + setattr(parser.values, option.dest, True) + builder.use_manylinux() + elif option_str in ('--no-manylinux', '--no-use-manylinux'): + setattr(parser.values, option.dest, False) + builder.no_use_manylinux() else: raise OptionValueError @@ -225,6 +231,16 @@ def configure_clp_pex_resolution(parser, builder): callback_args=(builder,), help='Whether to allow building of distributions from source; Default: allow builds') + group.add_option( + '--manylinux', '--no-manylinux', '--no-use-manylinux', + dest='use_manylinux', + default=True, + action='callback', + callback=process_precedence, + callback_args=(builder,), + help=('Whether to allow resolution of manylinux dists for linux target ' + 'platforms; Default: allow manylinux')) + # Set the pex tool to fetch from PyPI by default if nothing is specified. parser.set_default('repos', [PyPIFetcher()]) parser.add_option_group(group) @@ -324,18 +340,18 @@ def configure_clp_pex_environment(parser): group.add_option( '--platform', - dest='platform', + dest='platforms', default=[], type=str, action='append', help='The platform for which to build the PEX. This option can be passed multiple times ' - 'to create a multi-platform compatible pex. To build manylinux wheels for specific ' - 'tags, you can add them to the platform with hyphens like PLATFORM-PYVER-IMPL-ABI, ' - 'where PLATFORM is either "manylinux1-x86_64" or "manylinux1-i686", PYVER is a two-' - 'digit string representing the python version (e.g., 36), IMPL is the python ' - 'implementation abbreviation (e.g., cp, pp, jp), and ABI is the ABI tag (e.g., ' - 'cp36m, cp27mu, abi3, none). For example: manylinux1_x86_64-36-cp-cp36m. ' - 'Default: current platform.') + 'to create a multi-platform pex. To use wheels for specific interpreter/platform tags' + 'platform tags, you can append them to the platform with hyphens like: ' + 'PLATFORM-IMPL-PYVER-ABI (e.g. "linux_x86_64-cp-27-cp27mu", "macosx_10.12_x86_64-cp-36' + '-cp36m") PLATFORM is the host platform e.g. "linux-x86_64", "macosx-10.12-x86_64", etc' + '". IMPL is the python implementation abbreviation (e.g. "cp", "pp", "jp"). PYVER is ' + 'a two-digit string representing the python version (e.g. "27", "36"). ABI is the ABI ' + 'tag (e.g. "cp36m", "cp27mu", "abi3", "none"). Default: current platform.') group.add_option( '--interpreter-cache-dir', @@ -599,10 +615,11 @@ def build_pex(args, options, resolver_option_builder): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, - platforms=options.platform, + platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, - allow_prereleases=resolver_option_builder.prereleases_allowed) + allow_prereleases=resolver_option_builder.prereleases_allowed, + use_manylinux=options.use_manylinux) for dist in resolveds: log(' %s' % dist, v=options.verbosity) @@ -639,6 +656,14 @@ def transform_legacy_arg(arg): return arg +def _compatible_with_current_platform(platforms): + return ( + not platforms or + 'current' in platforms or + str(Platform.current()) in platforms + ) + + def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] @@ -676,8 +701,8 @@ def main(args=None): os.rename(tmp_name, options.pex_name) return 0 - if options.platform and get_platform() not in options.platform: - log('WARNING: attempting to run PEX with incompatible platforms!') + if not _compatible_with_current_platform(options.platforms): + log('WARNING: attempting to run PEX with incompatible platforms!', v=1) pex_builder.freeze() diff --git a/pex/environment.py b/pex/environment.py index 465675588..dad0fc036 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -20,10 +20,9 @@ from .common import die, open_zip, safe_mkdir, safe_rmtree from .interpreter import PythonInterpreter from .package import distribution_compatible -from .pep425tags import get_platform, get_supported from .pex_builder import PEXBuilder from .pex_info import PexInfo -from .resolver import platform_to_tags +from .platforms import Platform from .tracer import TRACER from .util import CacheHelper, DistributionHelper @@ -112,16 +111,26 @@ def load_internal_cache(cls, pex, pex_info): for dist in itertools.chain(*cls.write_zipped_internal_cache(pex, pex_info)): yield dist - def __init__(self, pex, pex_info, supported_tags=None, **kw): + def __init__(self, pex, pex_info, interpreter=None, **kw): self._internal_cache = os.path.join(pex, pex_info.internal_cache) self._pex = pex self._pex_info = pex_info self._activated = False self._working_set = None + self._interpreter = interpreter or PythonInterpreter.get() self._inherit_path = pex_info.inherit_path - self._supported_tags = supported_tags or get_supported() + self._supported_tags = [] super(PEXEnvironment, self).__init__( - search_path=[] if pex_info.inherit_path == 'false' else sys.path, **kw) + search_path=[] if pex_info.inherit_path == 'false' else sys.path, + **kw + ) + self._supported_tags.extend( + Platform.create(self.platform).supported_tags(self._interpreter) + ) + TRACER.log( + 'E: tags for %r x %r -> %s' % (self.platform, self._interpreter, self._supported_tags), + V=9 + ) def update_candidate_distributions(self, distribution_iter): for dist in distribution_iter: @@ -163,7 +172,6 @@ def _resolve(self, working_set, reqs): unresolved_reqs = set([req.lower() for req in unresolved_reqs]) if unresolved_reqs: - platform_str = '-'.join(platform_to_tags(get_platform(), PythonInterpreter.get())) TRACER.log('Unresolved requirements:') for req in unresolved_reqs: TRACER.log(' - %s' % req) @@ -174,8 +182,12 @@ def _resolve(self, working_set, reqs): for dist in self._pex_info.distributions: TRACER.log(' - %s' % dist) if not self._pex_info.ignore_errors: - die('Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % ( - platform_str, '\n'.join(map(str, unresolved_reqs)))) + die( + 'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % ( + Platform.current(), + '\n'.join(str(r) for r in unresolved_reqs) + ) + ) return resolveds diff --git a/pex/interpreter.py b/pex/interpreter.py index 3c10dde8a..5de9244a7 100644 --- a/pex/interpreter.py +++ b/pex/interpreter.py @@ -31,22 +31,25 @@ except ImportError: Integral = (int, long) -# Determine in the most platform-compatible way possible the identity of the interpreter -# and its known packages. -ID_PY = (b""" + +ID_PY_TMPL = b"""\ import sys import sysconfig import warnings -""" + -b'\n'.join(getsource(func).encode('utf-8') - for func in (get_flag, get_config_var, get_abbr_impl, - get_abi_tag, get_impl_version_info, get_impl_ver)) + -b""" -print("%s %s %s" % ( - get_abbr_impl(), - get_abi_tag(), - get_impl_ver())) +__CODE__ + + +print( + "%s %s %s %s %s %s" % ( + get_abbr_impl(), + get_abi_tag(), + get_impl_ver(), + sys.version_info[0], + sys.version_info[1], + sys.version_info[2] + ) +) setuptools_path = None try: @@ -63,7 +66,25 @@ rs = requirement_str.split('==', 2) if len(rs) == 2: print('%s %s %s' % (rs[0], rs[1], location)) -""") +""" + + +def _generate_identity_source(): + # Determine in the most platform-compatible way possible the identity of the interpreter + # and its known packages. + encodables = ( + get_flag, + get_config_var, + get_abbr_impl, + get_abi_tag, + get_impl_version_info, + get_impl_ver + ) + + return ID_PY_TMPL.replace( + b'__CODE__', + b'\n\n'.join(getsource(func).encode('utf-8') for func in encodables) + ) class PythonIdentity(object): @@ -88,22 +109,31 @@ class UnknownRequirement(Error): pass @classmethod def get(cls): - return cls(get_abbr_impl(), get_abi_tag(), get_impl_ver()) + return cls( + get_abbr_impl(), + get_abi_tag(), + get_impl_ver(), + str(sys.version_info[0]), + str(sys.version_info[1]), + str(sys.version_info[2]) + ) @classmethod def from_id_string(cls, id_string): + TRACER.log('creating PythonIdentity from id string: %s' % id_string, V=3) values = str(id_string).split() - if len(values) != 3: + if len(values) != 6: raise cls.InvalidError("Invalid id string: %s" % id_string) return cls(*values) - def __init__(self, abbr, abi, version): - self._interpreter = self.ABBR_TO_INTERPRETER[abbr] - self._abbr = abbr - self._version = tuple(int(x) for x in version) - if len(self._version) == 2: - self._version += (0,) - self._impl_ver = version + def __init__(self, impl, abi, impl_version, major, minor, patch): + assert impl in self.ABBR_TO_INTERPRETER, ( + 'unknown interpreter: {}'.format(impl) + ) + self._interpreter = self.ABBR_TO_INTERPRETER[impl] + self._abbr = impl + self._version = tuple(int(v) for v in (major, minor, patch)) + self._impl_ver = impl_version self._abi = abi @property @@ -177,13 +207,14 @@ def python(self): return '%d.%d' % (self.version[0:2]) def pkg_resources_env(self, platform_str): - """A dict that can be used in place of packaging.default_environment""" + """Returns a dict that can be used in place of packaging.default_environment.""" os_name = '' platform_machine = '' platform_release = '' platform_system = '' platform_version = '' sys_platform = '' + if 'win' in platform_str: os_name = 'nt' platform_machine = 'AMD64' if '64' in platform_str else 'x86' @@ -203,27 +234,36 @@ def pkg_resources_env(self, platform_str): platform_system = 'Darwin' platform_version = 'Darwin Kernel Version {}'.format(platform_release) sys_platform = 'darwin' + return { - "implementation_name": self.interpreter.lower(), - "implementation_version": self.version_str, - "os_name": os_name, - "platform_machine": platform_machine, - "platform_release": platform_release, - "platform_system": platform_system, - "platform_version": platform_version, - "python_full_version": self.version_str, - "platform_python_implementation": self.interpreter, - "python_version": self.version_str[:3], - "sys_platform": sys_platform, + 'implementation_name': self.interpreter.lower(), + 'implementation_version': self.version_str, + 'os_name': os_name, + 'platform_machine': platform_machine, + 'platform_release': platform_release, + 'platform_system': platform_system, + 'platform_version': platform_version, + 'python_full_version': self.version_str, + 'platform_python_implementation': self.interpreter, + 'python_version': self.version_str[:3], + 'sys_platform': sys_platform, } def __str__(self): - return '%s-%s.%s.%s' % (self._interpreter, - self._version[0], self._version[1], self._version[2]) + return '%s-%s.%s.%s' % ( + self._interpreter, + self._version[0], + self._version[1], + self._version[2] + ) def __repr__(self): return 'PythonIdentity(%r, %s, %s, %s)' % ( - self._interpreter, self._version[0], self._version[1], self._version[2]) + self._interpreter, + self._version[0], + self._version[1], + self._version[2] + ) def __eq__(self, other): return all([isinstance(other, PythonIdentity), @@ -294,7 +334,7 @@ def iter_extras(): def _from_binary_external(cls, binary, path_extras): environ = cls.sanitized_environment() environ['PYTHONPATH'] = ':'.join(path_extras) - stdout, _ = Executor.execute([binary], env=environ, stdin_payload=ID_PY) + stdout, _ = Executor.execute([binary], env=environ, stdin_payload=_generate_identity_source()) output = stdout.splitlines() if len(output) == 0: raise cls.IdentificationError('Could not establish identity of %s' % binary) diff --git a/pex/pep425tags.py b/pex/pep425tags.py index f1e1b4265..fe138452a 100644 --- a/pex/pep425tags.py +++ b/pex/pep425tags.py @@ -1,8 +1,7 @@ -# This file was copied from the pip project master branch on 2016/12/05 -# It was modified slightly to change a few import paths, remove the -# OrderedDict dependency, remove unneeded globals, and change get_supported to -# expect a single version. +# This file was forked from the pip project master branch on 2016/12/05 + """Generate and work with PEP 425 Compatibility Tags.""" + from __future__ import absolute_import import distutils.util @@ -125,26 +124,26 @@ def get_platform(): release, _, machine = platform.mac_ver() split_ver = release.split('.') - if machine == "x86_64" and _is_running_32bit(): - machine = "i386" - elif machine == "ppc64" and _is_running_32bit(): - machine = "ppc" + if machine == 'x86_64' and _is_running_32bit(): + machine = 'i386' + elif machine == 'ppc64' and _is_running_32bit(): + machine = 'ppc' return 'macosx_{0}_{1}_{2}'.format(split_ver[0], split_ver[1], machine) # XXX remove distutils dependency result = distutils.util.get_platform().replace('.', '_').replace('-', '_') - if result == "linux_x86_64" and _is_running_32bit(): + if result == 'linux_x86_64' and _is_running_32bit(): # 32 bit Python program (running on a 64 bit Linux): pip should only # install and run 32 bit compiled extensions in that case. - result = "linux_i686" + result = 'linux_i686' return result def is_manylinux1_compatible(): # Only Linux, and only x86-64 / i686 - if get_platform() not in ("linux_x86_64", "linux_i686"): + if get_platform() not in ('linux_x86_64', 'linux_i686'): return False # Check for presence of _manylinux module @@ -178,11 +177,6 @@ def _supports_arch(major, minor, arch): # 10.6 - Drops support for ppc64 # 10.7 - Drops support for ppc # - # Given that we do not know if we're installing a CLI or a GUI - # application, we must be conservative and assume it might be a GUI - # application and behave as if ppc64 and x86_64 support did not occur - # until 10.5. - # # Note: The above information is taken from the "Application support" # column in the chart not the "Processor support" since I believe # that we care about what instruction sets an application can use @@ -194,7 +188,7 @@ def _supports_arch(major, minor, arch): if arch == 'i386': return (major, minor) >= (10, 4) if arch == 'x86_64': - return (major, minor) >= (10, 5) + return (major, minor) >= (10, 4) if arch in groups: for garch in groups_dict[arch]: if _supports_arch(major, minor, garch): @@ -202,10 +196,10 @@ def _supports_arch(major, minor, arch): return False groups = ('fat', 'intel', 'fat64', 'fat32') - groups_dict = {"fat": ("i386", "ppc"), - "intel": ("x86_64", "i386"), - "fat64": ("x86_64", "ppc64"), - "fat32": ("x86_64", "i386", "ppc")} + groups_dict = {'fat': ('i386', 'ppc'), + 'intel': ('x86_64', 'i386'), + 'fat64': ('x86_64', 'ppc64'), + 'fat32': ('x86_64', 'i386', 'ppc')} if _supports_arch(major, minor, machine): arches.append(machine) @@ -219,10 +213,10 @@ def _supports_arch(major, minor, arch): return arches -def get_supported(version=None, noarch=False, platform=None, - impl=None, abi=None): +def get_supported(version=None, noarch=False, platform=None, impl=None, abi=None, + force_manylinux=False): """Return a list of supported tags for each version specified in - `versions`. + `version`. :param version: string version (e.g., "33", "32") or None. If None, use local system Python version. @@ -232,6 +226,8 @@ def get_supported(version=None, noarch=False, platform=None, tags for, or None. If None, use the local interpreter impl. :param abi: specify the exact abi you want valid tags for, or None. If None, use the local interpreter abi. + :param force_manylinux: Whether or not to force manylinux support. This is useful + when resolving for different target platform than current. """ supported = [] @@ -279,7 +275,12 @@ def get_supported(version=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif platform is None and is_manylinux1_compatible(): + elif ( + (platform is None and is_manylinux1_compatible()) or + # N.B. Here we work around the fact that `is_manylinux1_compatible()` expects + # to be running on the target platform being built for with a feature flag approach. + (arch.startswith('linux') and force_manylinux) + ): arches = [arch.replace('linux', 'manylinux1'), arch] else: arches = [arch] @@ -296,7 +297,7 @@ def get_supported(version=None, noarch=False, platform=None, break for abi in abi3s: # empty set if not Python 3 for arch in arches: - supported.append(("%s%s" % (impl, version), abi, arch)) + supported.append(('%s%s' % (impl, version), abi, arch)) # Has binaries, does not use the Python API: for arch in arches: diff --git a/pex/platforms.py b/pex/platforms.py new file mode 100644 index 000000000..b05edae45 --- /dev/null +++ b/pex/platforms.py @@ -0,0 +1,80 @@ +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +from collections import namedtuple + +from .pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver, get_platform, get_supported + + +class Platform(namedtuple('Platform', ['platform', 'impl', 'version', 'abi'])): + """Represents a target platform and it's extended interpreter compatibility + tags (e.g. implementation, version and ABI).""" + + SEP = '-' + + def __new__(cls, platform, impl=None, version=None, abi=None): + platform = platform.replace('-', '_').replace('.', '_') + if all((impl, version, abi)): + abi = cls._maybe_prefix_abi(impl, version, abi) + return super(cls, Platform).__new__(cls, platform, impl, version, abi) + + def __str__(self): + return self.SEP.join(self) if all(self) else self.platform + + @classmethod + def current(cls): + platform = get_platform() + impl = get_abbr_impl() + version = get_impl_ver() + abi = get_abi_tag() + return cls(platform, impl, version, abi) + + @classmethod + def create(cls, platform): + if isinstance(platform, Platform): + return platform + + platform = platform.lower() + if platform == 'current': + return cls.current() + + try: + platform, impl, version, abi = platform.rsplit(cls.SEP, 3) + except ValueError: + return cls(platform) + else: + return cls(platform, impl, version, abi) + + @staticmethod + def _maybe_prefix_abi(impl, version, abi): + # N.B. This permits users to pass in simpler extended platform strings like + # `linux-x86_64-cp-27-mu` vs e.g. `linux-x86_64-cp-27-cp27mu`. + impl_ver = ''.join((impl, version)) + return abi if abi.startswith(impl_ver) else ''.join((impl_ver, abi)) + + @property + def is_extended(self): + return all(attr is not None for attr in (self.impl, self.version, self.abi)) + + def supported_tags(self, interpreter=None, force_manylinux=True): + """Returns a list of supported PEP425 tags for the current platform.""" + if interpreter and not self.is_extended: + tags = get_supported( + platform=self.platform, + impl=interpreter.identity.abbr_impl, + version=interpreter.identity.impl_ver, + abi=interpreter.identity.abi_tag, + force_manylinux=force_manylinux + ) + else: + tags = get_supported( + platform=self.platform, + impl=self.impl, + version=self.version, + abi=self.abi, + force_manylinux=force_manylinux + ) + + return tags diff --git a/pex/resolver.py b/pex/resolver.py index 4520e2629..4aaa13780 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -15,11 +15,11 @@ from .common import safe_mkdir from .fetcher import Fetcher -from .interpreter import PythonIdentity, PythonInterpreter +from .interpreter import PythonInterpreter from .iterator import Iterator, IteratorInterface from .orderedset import OrderedSet from .package import Package, distribution_compatible -from .pep425tags import get_platform, get_supported +from .platforms import Platform from .resolvable import ResolvableRequirement, resolvables_from_iterable from .resolver_options import ResolverOptionsBuilder from .tracer import TRACER @@ -117,8 +117,12 @@ def _check(self): # Check whether or not the resolvables in this set are satisfiable, raise an exception if not. for name, resolved_packages in self._collapse().items(): if not resolved_packages.packages: - raise Unsatisfiable('Could not satisfy all requirements for %s:\n %s' % ( - resolved_packages.resolvable, self._synthesize_parents(name))) + raise Unsatisfiable( + 'Could not satisfy all requirements for %s:\n %s' % ( + resolved_packages.resolvable, + self._synthesize_parents(name) + ) + ) def merge(self, resolvable, packages, parent=None): """Add a resolvable and its resolved packages.""" @@ -160,23 +164,26 @@ class Resolver(object): class Error(Exception): pass - def filter_packages_by_interpreter(self, packages): - return [package for package in packages - if package.compatible(self._supported_tags)] - def __init__(self, allow_prereleases=None, interpreter=None, platform=None, - pkg_blacklist=None): + pkg_blacklist=None, use_manylinux=None): self._interpreter = interpreter or PythonInterpreter.get() - self._platform = platform or get_platform() - # Turn a platform string into something we can actually use - platform_tag, version, impl, abi = platform_to_tags(self._platform, self._interpreter) - self._identity = PythonIdentity(impl, abi, version) + self._platform = Platform.create(platform) if platform else Platform.current() self._allow_prereleases = allow_prereleases self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {} - self._supported_tags = get_supported(version=version, - platform=platform_tag, - impl=impl, - abi=abi) + self._supported_tags = self._platform.supported_tags( + self._interpreter, + use_manylinux + ) + TRACER.log( + 'R: tags for %r x %r -> %s' % (self._platform, self._interpreter, self._supported_tags), + V=9 + ) + + def filter_packages_by_supported_tags(self, packages, supported_tags=None): + return [ + package for package in packages + if package.compatible(supported_tags or self._supported_tags) + ] def package_iterator(self, resolvable, existing=None): if existing: @@ -184,7 +191,7 @@ def package_iterator(self, resolvable, existing=None): StaticIterator(existing, allow_prereleases=self._allow_prereleases)) else: existing = resolvable.packages() - return self.filter_packages_by_interpreter(existing) + return self.filter_packages_by_supported_tags(existing) def build(self, package, options): context = options.get_context() @@ -249,10 +256,11 @@ def resolve(self, resolvables, resolvable_set=None): new_parent = '%s->%s' % (parent, resolvable) if parent else str(resolvable) # We patch packaging.markers.default_environment here so we find optional reqs for the # platform we're building the PEX for, rather than the one we're on. - with patched_packing_env(self._identity.pkg_resources_env(self._platform)): + with patched_packing_env(self._interpreter.identity.pkg_resources_env(self._platform)): resolvables.extend( (ResolvableRequirement(req, resolvable.options), new_parent) for req in - distribution.requires(extras=resolvable_set.extras(resolvable.name))) + distribution.requires(extras=resolvable_set.extras(resolvable.name)) + ) resolvable_set = resolvable_set.replace_built(built_packages) # We may have built multiple distributions depending upon if we found transitive dependencies @@ -292,7 +300,7 @@ def __init__(self, cache, cache_ttl, *args, **kw): def package_iterator(self, resolvable, existing=None): iterator = Iterator(fetchers=[Fetcher([self.__cache])], allow_prereleases=self._allow_prereleases) - packages = self.filter_packages_by_interpreter(resolvable.compatible(iterator)) + packages = self.filter_packages_by_supported_tags(resolvable.compatible(iterator)) if packages and self.__cache_ttl: packages = self.filter_packages_by_ttl(packages, self.__cache_ttl) @@ -346,7 +354,8 @@ def resolve(requirements, cache=None, cache_ttl=None, allow_prereleases=None, - pkg_blacklist=None): + pkg_blacklist=None, + use_manylinux=None): """Produce all distributions needed to (recursively) meet `requirements` :param requirements: An iterator of Requirement-like things, either @@ -389,6 +398,7 @@ def resolve(requirements, For example, a valid blacklist is {'functools32': 'CPython>3'}. NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution tracked by: https://github.com/pantsbuild/pex/issues/456 + :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves. :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for @@ -419,6 +429,7 @@ def resolve(requirements, builder = ResolverOptionsBuilder(fetchers=fetchers, allow_prereleases=allow_prereleases, + use_manylinux=use_manylinux, precedence=precedence, context=context) @@ -426,11 +437,13 @@ def resolve(requirements, resolver = CachingResolver(cache, cache_ttl, allow_prereleases=allow_prereleases, + use_manylinux=use_manylinux, interpreter=interpreter, platform=platform, pkg_blacklist=pkg_blacklist) else: resolver = Resolver(allow_prereleases=allow_prereleases, + use_manylinux=use_manylinux, interpreter=interpreter, platform=platform, pkg_blacklist=pkg_blacklist) @@ -447,7 +460,8 @@ def resolve_multi(requirements, cache=None, cache_ttl=None, allow_prereleases=None, - pkg_blacklist=None): + pkg_blacklist=None, + use_manylinux=None): """A generator function that produces all distributions needed to meet `requirements` for multiple interpreters and/or platforms. @@ -493,7 +507,7 @@ def resolve_multi(requirements, """ interpreters = interpreters or [PythonInterpreter.get()] - platforms = platforms or [get_platform()] + platforms = platforms or [Platform.current()] seen = set() for interpreter in interpreters: @@ -507,7 +521,8 @@ def resolve_multi(requirements, cache, cache_ttl, allow_prereleases, - pkg_blacklist=pkg_blacklist): + pkg_blacklist=pkg_blacklist, + use_manylinux=use_manylinux): if resolvable not in seen: seen.add(resolvable) yield resolvable diff --git a/pex/resolver_options.py b/pex/resolver_options.py index 8cf3a3514..4afc7b920 100644 --- a/pex/resolver_options.py +++ b/pex/resolver_options.py @@ -17,19 +17,19 @@ class ResolverOptionsInterface(object): def get_context(self): - raise NotImplemented + raise NotImplementedError def get_crawler(self): - raise NotImplemented + raise NotImplementedError def get_sorter(self): - raise NotImplemented + raise NotImplementedError def get_translator(self, interpreter, supported_tags): - raise NotImplemented + raise NotImplementedError def get_iterator(self): - raise NotImplemented + raise NotImplementedError class ResolverOptionsBuilder(object): @@ -44,6 +44,7 @@ def __init__(self, allow_external=None, allow_unverified=None, allow_prereleases=None, + use_manylinux=None, precedence=None, context=None): self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()] @@ -53,6 +54,7 @@ def __init__(self, self._allow_prereleases = allow_prereleases self._precedence = precedence if precedence is not None else Sorter.DEFAULT_PACKAGE_PRECEDENCE self._context = context or Context.get() + self._use_manylinux = use_manylinux def clone(self): return ResolverOptionsBuilder( @@ -61,6 +63,7 @@ def clone(self): allow_external=self._allow_external.copy(), allow_unverified=self._allow_unverified.copy(), allow_prereleases=self._allow_prereleases, + use_manylinux=self._use_manylinux, precedence=self._precedence[:], context=self._context, ) @@ -107,6 +110,14 @@ def no_use_wheel(self): [precedent for precedent in self._precedence if precedent is not WheelPackage]) return self + def use_manylinux(self): + self._use_manylinux = True + return self + + def no_use_manylinux(self): + self._use_manylinux = False + return self + def allow_builds(self): if SourcePackage not in self._precedence: self._precedence = self._precedence + (SourcePackage,) @@ -152,6 +163,7 @@ def build(self, key): allow_external=self._allow_all_external or key in self._allow_external, allow_unverified=key in self._allow_unverified, allow_prereleases=self._allow_prereleases, + use_manylinux=self._use_manylinux, precedence=self._precedence, context=self._context, ) @@ -163,12 +175,14 @@ def __init__(self, allow_external=False, allow_unverified=False, allow_prereleases=None, + use_manylinux=None, precedence=None, context=None): self._fetchers = fetchers if fetchers is not None else [PyPIFetcher()] self._allow_external = allow_external self._allow_unverified = allow_unverified self._allow_prereleases = allow_prereleases + self._use_manylinux = use_manylinux self._precedence = precedence if precedence is not None else Sorter.DEFAULT_PACKAGE_PRECEDENCE self._context = context or Context.get() diff --git a/pex/testing.py b/pex/testing.py index b02ccd342..da31c49ff 100644 --- a/pex/testing.py +++ b/pex/testing.py @@ -21,12 +21,13 @@ from .util import DistributionHelper, named_temporary_file from .version import SETUPTOOLS_REQUIREMENT -NOT_CPYTHON_36 = ( - "hasattr(sys, 'pypy_version_info') or " - "(sys.version_info[0], sys.version_info[1]) != (3, 6)" -) - -PYPY = "hasattr(sys, 'pypy_version_info')" +IS_PYPY = "hasattr(sys, 'pypy_version_info')" +NOT_CPYTHON27 = ("%s or (sys.version_info[0], sys.version_info[1]) != (2, 7)" % (IS_PYPY)) +NOT_CPYTHON36 = ("%s or (sys.version_info[0], sys.version_info[1]) != (3, 6)" % (IS_PYPY)) +IS_LINUX = "platform.system() == 'Linux'" +IS_NOT_LINUX = "platform.system() != 'Linux'" +NOT_CPYTHON27_OR_OSX = "%s or %s" % (NOT_CPYTHON27, IS_NOT_LINUX) +NOT_CPYTHON36_OR_LINUX = "%s or %s" % (NOT_CPYTHON36, IS_LINUX) @contextlib.contextmanager @@ -217,10 +218,12 @@ class IntegResults(namedtuple('results', 'output return_code exception traceback """Convenience object to return integration run results.""" def assert_success(self): - if not (self.exception is None and self.return_code is None): - raise AssertionError('integration test failed: return_code=%s, exception=%r, traceback=%s' % ( - self.return_code, self.exception, self.traceback - )) + if not (self.exception is None and self.return_code in [None, 0]): + raise AssertionError( + 'integration test failed: return_code=%s, exception=%r, output=%s, traceback=%s' % ( + self.return_code, self.exception, self.output, self.traceback + ) + ) def assert_failure(self): assert self.exception or self.return_code diff --git a/tests/test_integration.py b/tests/test_integration.py index a5a87e6ec..903104ef3 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,8 +1,12 @@ # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). +import functools import os +import platform +import subprocess import sys +from contextlib import contextmanager from textwrap import dedent import pytest @@ -12,8 +16,10 @@ from pex.installer import EggInstaller from pex.pex_bootstrapper import get_pex_info from pex.testing import ( - NOT_CPYTHON_36, - PYPY, + IS_PYPY, + NOT_CPYTHON27_OR_OSX, + NOT_CPYTHON36, + NOT_CPYTHON36_OR_LINUX, ensure_python_interpreter, get_dep_dist_names_from_pex, run_pex_command, @@ -167,7 +173,9 @@ def do_something(): assert rc == 1 -@pytest.mark.skipif(NOT_CPYTHON_36) +# TODO: https://github.com/pantsbuild/pex/issues/479 +@pytest.mark.skipif(NOT_CPYTHON36_OR_LINUX, + reason='inherits linux abi on linux w/ no backing packages') def test_pex_multi_resolve(): """Tests multi-interpreter + multi-platform resolution.""" with temporary_dir() as output_dir: @@ -175,7 +183,7 @@ def test_pex_multi_resolve(): results = run_pex_command(['--disable-cache', 'lxml==3.8.0', '--no-build', - '--platform=manylinux1-x86_64', + '--platform=linux-x86_64', '--platform=macosx-10.6-x86_64', '--python=python2.7', '--python=python3.6', @@ -341,7 +349,7 @@ def test_interpreter_constraints_to_pex_info_py2(): assert set(['>=2.7', '<3']) == set(pex_info.interpreter_constraints) -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_interpreter_constraints_to_pex_info_py3(): py3_interpreter = ensure_python_interpreter('3.6.3') with environment_as(PATH=os.path.dirname(py3_interpreter)): @@ -369,7 +377,7 @@ def test_interpreter_resolution_with_constraint_option(): assert pex_info.build_properties['version'][0] < 3 -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_interpreter_resolution_with_pex_python_path(): with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') @@ -404,7 +412,7 @@ def test_interpreter_resolution_with_pex_python_path(): assert str(pex_python_path.split(':')[0]).encode() in stdout -@pytest.mark.skipif(NOT_CPYTHON_36) +@pytest.mark.skipif(NOT_CPYTHON36) def test_interpreter_resolution_pex_python_path_precedence_over_pex_python(): with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') @@ -446,7 +454,7 @@ def test_plain_pex_exec_no_ppp_no_pp_no_constraints(): assert str(sys.executable).encode() in stdout -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_pex_exec_with_pex_python_path_only(): with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') @@ -472,7 +480,7 @@ def test_pex_exec_with_pex_python_path_only(): assert str(pex_python_path.split(':')[0]).encode() in stdout -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): with temporary_dir() as td: pexrc_path = os.path.join(td, '.pexrc') @@ -500,7 +508,7 @@ def test_pex_exec_with_pex_python_path_and_pex_python_but_no_constraints(): assert str(pex_python_path.split(':')[0]).encode() in stdout -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_pex_python(): py2_path_interpreter = ensure_python_interpreter('2.7.10') py3_path_interpreter = ensure_python_interpreter('3.6.3') @@ -561,7 +569,7 @@ def test_pex_python(): assert correct_interpreter_path in stdout -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_entry_point_targeting(): """Test bugfix for https://github.com/pantsbuild/pex/issues/434""" with temporary_dir() as td: @@ -582,7 +590,7 @@ def test_entry_point_targeting(): assert 'usage: autopep8'.encode() in stdout -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_interpreter_selection_using_os_environ_for_bootstrap_reexec(): """ This is a test for verifying the proper function of the @@ -698,6 +706,7 @@ def inherit_path(inherit_path): '-o', pex_path, ]) + results.assert_success() env = os.environ.copy() @@ -719,3 +728,105 @@ def inherit_path(inherit_path): assert requests_paths[0] < sys_paths[0] else: assert requests_paths[0] > sys_paths[0] + + +def test_pex_multi_resolve_2(): + """Tests multi-interpreter + multi-platform resolution using extended platform notation.""" + with temporary_dir() as output_dir: + pex_path = os.path.join(output_dir, 'pex.pex') + results = run_pex_command(['--disable-cache', + 'lxml==3.8.0', + '--no-build', + '--platform=linux-x86_64-cp-36-m', + '--platform=linux-x86_64-cp-27-m', + '--platform=macosx-10.6-x86_64-cp-36-m', + '--platform=macosx-10.6-x86_64-cp-27-m', + '-o', pex_path]) + results.assert_success() + + included_dists = get_dep_dist_names_from_pex(pex_path, 'lxml') + assert len(included_dists) == 4 + for dist_substr in ('-cp27-', '-cp36-', '-manylinux1_x86_64', '-macosx_'): + assert any(dist_substr in f for f in included_dists), ( + '{} was not found in wheel'.format(dist_substr) + ) + + +@contextmanager +def pex_manylinux_and_tag_selection_context(): + with temporary_dir() as output_dir: + def do_resolve(req_name, req_version, platform, extra_flags=None): + extra_flags = extra_flags or '' + pex_path = os.path.join(output_dir, 'test.pex') + results = run_pex_command(['--disable-cache', + '--no-build', + '%s==%s' % (req_name, req_version), + '--platform=%s' % (platform), + '-o', pex_path] + extra_flags.split()) + return pex_path, results + + def test_resolve(req_name, req_version, platform, substr, extra_flags=None): + pex_path, results = do_resolve(req_name, req_version, platform, extra_flags) + results.assert_success() + included_dists = get_dep_dist_names_from_pex(pex_path, req_name.replace('-', '_')) + assert any( + substr in d for d in included_dists + ), 'couldnt find {} in {}'.format(substr, included_dists) + + def ensure_failure(req_name, req_version, platform, extra_flags): + pex_path, results = do_resolve(req_name, req_version, platform, extra_flags) + results.assert_failure() + + yield test_resolve, ensure_failure + + +@pytest.mark.skipif(IS_PYPY) +def test_pex_manylinux_and_tag_selection_linux_msgpack(): + """Tests resolver manylinux support and tag targeting.""" + with pex_manylinux_and_tag_selection_context() as (test_resolve, ensure_failure): + msgpack, msgpack_ver = 'msgpack-python', '0.4.7' + test_msgpack = functools.partial(test_resolve, msgpack, msgpack_ver) + + # Exclude 3.3 and 3.6 because no 33/36 wheel exists on pypi. + if (sys.version_info[0], sys.version_info[1]) not in [(3, 3), (3, 6)]: + test_msgpack('linux-x86_64', 'manylinux1_x86_64.whl') + + test_msgpack('linux-x86_64-cp-27-m', 'msgpack_python-0.4.7-cp27-cp27m-manylinux1_x86_64.whl') + test_msgpack('linux-x86_64-cp-27-mu', 'msgpack_python-0.4.7-cp27-cp27mu-manylinux1_x86_64.whl') + test_msgpack('linux-i686-cp-27-m', 'msgpack_python-0.4.7-cp27-cp27m-manylinux1_i686.whl') + test_msgpack('linux-i686-cp-27-mu', 'msgpack_python-0.4.7-cp27-cp27mu-manylinux1_i686.whl') + test_msgpack('linux-x86_64-cp-27-mu', 'msgpack_python-0.4.7-cp27-cp27mu-manylinux1_x86_64.whl') + test_msgpack('linux-x86_64-cp-34-m', 'msgpack_python-0.4.7-cp34-cp34m-manylinux1_x86_64.whl') + test_msgpack('linux-x86_64-cp-35-m', 'msgpack_python-0.4.7-cp35-cp35m-manylinux1_x86_64.whl') + + ensure_failure(msgpack, msgpack_ver, 'linux-x86_64', '--no-manylinux') + + +def test_pex_manylinux_and_tag_selection_lxml_osx(): + with pex_manylinux_and_tag_selection_context() as (test_resolve, ensure_failure): + test_resolve('lxml', '3.8.0', 'macosx-10.6-x86_64-cp-27-m', 'lxml-3.8.0-cp27-cp27m-macosx') + test_resolve('lxml', '3.8.0', 'macosx-10.6-x86_64-cp-36-m', 'lxml-3.8.0-cp36-cp36m-macosx') + + +@pytest.mark.skipif(NOT_CPYTHON27_OR_OSX) +def test_pex_manylinux_runtime(): + """Tests resolver manylinux support and runtime resolution (and --platform=current).""" + test_stub = dedent( + """ + import msgpack + print(msgpack.unpackb(msgpack.packb([1, 2, 3]))) + """ + ) + + with temporary_content({'tester.py': test_stub}) as output_dir: + pex_path = os.path.join(output_dir, 'test.pex') + tester_path = os.path.join(output_dir, 'tester.py') + results = run_pex_command(['--disable-cache', + '--no-build', + 'msgpack-python==0.4.7', + '--platform=current'.format(platform), + '-o', pex_path]) + results.assert_success() + + out = subprocess.check_output([pex_path, tester_path]) + assert out.strip() == '[1, 2, 3]' diff --git a/tests/test_interpreter.py b/tests/test_interpreter.py index 8aa466111..cccf4a8df 100644 --- a/tests/test_interpreter.py +++ b/tests/test_interpreter.py @@ -6,6 +6,7 @@ import pytest from pex import interpreter +from pex.testing import IS_PYPY, ensure_python_interpreter try: from mock import patch @@ -21,3 +22,11 @@ def test_all_does_not_raise_with_empty_path_envvar(self): with patch.dict(os.environ, clear=True): reload(interpreter) interpreter.PythonInterpreter.all() + + @pytest.mark.skipif(IS_PYPY) + def test_interpreter_versioning(self): + test_version_tuple = (2, 7, 10) + test_version = '.'.join(str(x) for x in test_version_tuple) + test_interpreter = ensure_python_interpreter(test_version) + py_interpreter = interpreter.PythonInterpreter.from_binary(test_interpreter) + assert py_interpreter.identity.version == test_version_tuple diff --git a/tests/test_pep425tags.py b/tests/test_pep425tags.py new file mode 100644 index 000000000..fc7b89e24 --- /dev/null +++ b/tests/test_pep425tags.py @@ -0,0 +1,168 @@ +# This file was forked from the pip project master branch on 2016/12/05 + +import sys + +from pex import pep425tags + +try: + from mock import patch +except ImportError: + from unittest.mock import patch + + +class TestPEP425Tags(object): + + def mock_get_config_var(self, **kwd): + """ + Patch sysconfig.get_config_var for arbitrary keys. + """ + import pex.pep425tags + + get_config_var = pex.pep425tags.sysconfig.get_config_var + + def _mock_get_config_var(var): + if var in kwd: + return kwd[var] + return get_config_var(var) + return _mock_get_config_var + + def abi_tag_unicode(self, flags, config_vars): + """ + Used to test ABI tags, verify correct use of the `u` flag + """ + import pex.pep425tags + + config_vars.update({'SOABI': None}) + base = pex.pep425tags.get_abbr_impl() + pex.pep425tags.get_impl_ver() + + if sys.version_info < (3, 3): + config_vars.update({'Py_UNICODE_SIZE': 2}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('pex.pep425tags.sysconfig.get_config_var', mock_gcf): + abi_tag = pex.pep425tags.get_abi_tag() + assert abi_tag == base + flags + + config_vars.update({'Py_UNICODE_SIZE': 4}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('pex.pep425tags.sysconfig.get_config_var', mock_gcf): + abi_tag = pex.pep425tags.get_abi_tag() + assert abi_tag == base + flags + 'u' + + else: + # On Python >= 3.3, UCS-4 is essentially permanently enabled, and + # Py_UNICODE_SIZE is None. SOABI on these builds does not include + # the 'u' so manual SOABI detection should not do so either. + config_vars.update({'Py_UNICODE_SIZE': None}) + mock_gcf = self.mock_get_config_var(**config_vars) + with patch('pex.pep425tags.sysconfig.get_config_var', mock_gcf): + abi_tag = pex.pep425tags.get_abi_tag() + assert abi_tag == base + flags + + def test_broken_sysconfig(self): + """ + Test that pep425tags still works when sysconfig is broken. + Can be a problem on Python 2.7 + Issue #1074. + """ + import pex.pep425tags + + def raises_ioerror(var): + raise IOError("I have the wrong path!") + + with patch('pex.pep425tags.sysconfig.get_config_var', raises_ioerror): + assert len(pex.pep425tags.get_supported()) + + def test_no_hyphen_tag(self): + """ + Test that no tag contains a hyphen. + """ + import pex.pep425tags + + mock_gcf = self.mock_get_config_var(SOABI='cpython-35m-darwin') + + with patch('pex.pep425tags.sysconfig.get_config_var', mock_gcf): + supported = pex.pep425tags.get_supported() + + for (py, abi, plat) in supported: + assert '-' not in py + assert '-' not in abi + assert '-' not in plat + + def test_manual_abi_noflags(self): + """ + Test that no flags are set on a non-PyDebug, non-Pymalloc ABI tag. + """ + self.abi_tag_unicode('', {'Py_DEBUG': False, 'WITH_PYMALLOC': False}) + + def test_manual_abi_d_flag(self): + """ + Test that the `d` flag is set on a PyDebug, non-Pymalloc ABI tag. + """ + self.abi_tag_unicode('d', {'Py_DEBUG': True, 'WITH_PYMALLOC': False}) + + def test_manual_abi_m_flag(self): + """ + Test that the `m` flag is set on a non-PyDebug, Pymalloc ABI tag. + """ + self.abi_tag_unicode('m', {'Py_DEBUG': False, 'WITH_PYMALLOC': True}) + + def test_manual_abi_dm_flags(self): + """ + Test that the `dm` flags are set on a PyDebug, Pymalloc ABI tag. + """ + self.abi_tag_unicode('dm', {'Py_DEBUG': True, 'WITH_PYMALLOC': True}) + + +class TestManylinux1Tags(object): + + @patch('pex.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + def test_manylinux1_compatible_on_linux_x86_64(self): + """ + Test that manylinux1 is enabled on linux_x86_64 + """ + assert pep425tags.is_manylinux1_compatible() + + @patch('pex.pep425tags.get_platform', lambda: 'linux_i686') + @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + def test_manylinux1_compatible_on_linux_i686(self): + """ + Test that manylinux1 is enabled on linux_i686 + """ + assert pep425tags.is_manylinux1_compatible() + + @patch('pex.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: False) + def test_manylinux1_2(self): + """ + Test that manylinux1 is disabled with incompatible glibc + """ + assert not pep425tags.is_manylinux1_compatible() + + @patch('pex.pep425tags.get_platform', lambda: 'arm6vl') + @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + def test_manylinux1_3(self): + """ + Test that manylinux1 is disabled on arm6vl + """ + assert not pep425tags.is_manylinux1_compatible() + + @patch('pex.pep425tags.get_platform', lambda: 'linux_x86_64') + @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + @patch('sys.platform', 'linux2') + def test_manylinux1_tag_is_first(self): + """ + Test that the more specific tag manylinux1 comes first. + """ + groups = {} + for pyimpl, abi, arch in pep425tags.get_supported(): + groups.setdefault((pyimpl, abi), []).append(arch) + + for arches in groups.values(): + if arches == ['any']: + continue + # Expect the most specific arch first: + if len(arches) == 3: + assert arches == ['manylinux1_x86_64', 'linux_x86_64', 'any'] + else: + assert arches == ['manylinux1_x86_64', 'linux_x86_64'] diff --git a/tests/test_pex_binary.py b/tests/test_pex_binary.py index 4e1338dba..63f6f1810 100644 --- a/tests/test_pex_binary.py +++ b/tests/test_pex_binary.py @@ -160,6 +160,7 @@ def __init__(self, allow_external=None, allow_unverified=None, allow_prereleases=None, + use_manylinux=None, precedence=None, context=None ): @@ -168,6 +169,7 @@ def __init__(self, allow_external=allow_external, allow_unverified=allow_unverified, allow_prereleases=allow_prereleases, + use_manylinux=None, precedence=precedence, context=context) self._fetchers.insert(0, fetcher) diff --git a/tests/test_pex_bootstrapper.py b/tests/test_pex_bootstrapper.py index dd414e847..635f1c430 100644 --- a/tests/test_pex_bootstrapper.py +++ b/tests/test_pex_bootstrapper.py @@ -9,7 +9,7 @@ from pex.common import open_zip from pex.interpreter import PythonInterpreter from pex.pex_bootstrapper import find_compatible_interpreters, get_pex_info -from pex.testing import PYPY, ensure_python_interpreter, write_simple_pex +from pex.testing import IS_PYPY, ensure_python_interpreter, write_simple_pex def test_get_pex_info(): @@ -32,7 +32,7 @@ def test_get_pex_info(): assert pex_info.dump() == pex_info_2.dump() -@pytest.mark.skipif(PYPY) +@pytest.mark.skipif(IS_PYPY) def test_find_compatible_interpreters(): pex_python_path = ':'.join([ ensure_python_interpreter('2.7.9'), diff --git a/tests/test_platform.py b/tests/test_platform.py new file mode 100644 index 000000000..7ca1b8f80 --- /dev/null +++ b/tests/test_platform.py @@ -0,0 +1,86 @@ +# Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import sys + +import pytest + +from pex.pep425tags import get_abi_tag, get_impl_tag +from pex.platforms import Platform + +EXPECTED_BASE = [('py27', 'none', 'any'), ('py2', 'none', 'any')] + + +def test_platform(): + assert Platform('linux-x86_64', 'cp', '27', 'mu') == ('linux_x86_64', 'cp', '27', 'cp27mu') + assert str( + Platform('linux-x86_64', 'cp', '27', 'm') + ) == 'linux_x86_64-cp-27-cp27m' + assert str(Platform('linux-x86_64')) == 'linux_x86_64' + + +def test_platform_create(): + assert Platform.create('linux-x86_64') == ('linux_x86_64', None, None, None) + assert Platform.create('linux-x86_64-cp-27-cp27mu') == ('linux_x86_64', 'cp', '27', 'cp27mu') + assert Platform.create('linux-x86_64-cp-27-mu') == ('linux_x86_64', 'cp', '27', 'cp27mu') + assert Platform.create( + 'macosx-10.4-x86_64-cp-27-m') == ('macosx_10_4_x86_64', 'cp', '27', 'cp27m') + + +def test_platform_create_noop(): + existing = Platform.create('linux-x86_64') + assert Platform.create(existing) == existing + + +def test_platform_current(): + assert Platform.create('current') == Platform.current() + + +def assert_tags(platform, expected_tags, manylinux=None): + tags = Platform.create(platform).supported_tags(force_manylinux=manylinux) + for expected_tag in expected_tags: + assert expected_tag in tags + + +def test_platform_supported_tags_linux(): + assert_tags( + 'linux-x86_64-cp-27-mu', + EXPECTED_BASE + [('cp27', 'cp27mu', 'linux_x86_64')] + ) + + +def test_platform_supported_tags_manylinux(): + assert_tags( + 'linux-x86_64-cp-27-mu', + EXPECTED_BASE + [('cp27', 'cp27mu', 'manylinux1_x86_64')], + True + ) + + +@pytest.mark.skipif("(sys.version_info[0], sys.version_info[1]) == (2, 6)") +def test_platform_supported_tags_osx_minimal(): + assert_tags( + 'macosx-10.4-x86_64', + [ + (get_impl_tag(), 'none', 'any'), + ('py%s' % sys.version_info[0], 'none', 'any'), + (get_impl_tag(), get_abi_tag(), 'macosx_10_4_x86_64') + ] + ) + + +def test_platform_supported_tags_osx_full(): + assert_tags( + 'macosx-10.12-x86_64-cp-27-m', + EXPECTED_BASE + [ + ('cp27', 'cp27m', 'macosx_10_4_x86_64'), + ('cp27', 'cp27m', 'macosx_10_5_x86_64'), + ('cp27', 'cp27m', 'macosx_10_6_x86_64'), + ('cp27', 'cp27m', 'macosx_10_7_x86_64'), + ('cp27', 'cp27m', 'macosx_10_8_x86_64'), + ('cp27', 'cp27m', 'macosx_10_9_x86_64'), + ('cp27', 'cp27m', 'macosx_10_10_x86_64'), + ('cp27', 'cp27m', 'macosx_10_11_x86_64'), + ('cp27', 'cp27m', 'macosx_10_12_x86_64'), + ] + ) diff --git a/tox.ini b/tox.ini index 3c8cd405e..696eb86c2 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ envlist = [testenv] commands = - py.test {posargs:-vvsx} + py.test {posargs:-vvs} # Ensure pex's main entrypoint can be run externally. pex --cache-dir {envtmpdir}/buildcache wheel requests . -e pex.bin.pex:main --version deps =