WIP Feature/pip wheels windows

src/rez/cli/pip.py

@@ -4,9 +4,6 @@
 
 
 def setup_parser(parser, completions=False):
-    parser.add_argument(
-        "--pip-version", dest="pip_ver", metavar="VERSION",
-        help="pip version (rez package) to use, default is latest")
     parser.add_argument(
         "--python-version", dest="py_ver", metavar="VERSION",
         help="python version (rez package) to use, default is latest. Note "
@@ -29,7 +26,6 @@ def setup_parser(parser, completions=False):
 
 def command(opts, parser, extra_arg_groups=None):
     from rez.pip import pip_install_package, run_pip_command
-    import sys
 
     if not (opts.search or opts.install):
         parser.error("Expected one of: --install, --search")
@@ -41,7 +37,6 @@ def command(opts, parser, extra_arg_groups=None):
 
     installed_variants, skipped_variants = pip_install_package(
         opts.PACKAGE,
-        pip_version=opts.pip_ver,
         python_version=opts.py_ver,
         release=opts.release)
 

src/rez/pip.py

@@ -16,7 +16,6 @@
 from tempfile import mkdtemp
 from StringIO import StringIO
 from pipes import quote
-import subprocess
 import os.path
 import shutil
 import sys
@@ -34,12 +33,13 @@ class InstallMode(Enum):
     # dependency, if the dependency is newer.
     new_deps = 2
     # install dependencies even if a rez package of the same version is already
-    # available, if possible. For example, if you are performing a local install,
-    # a released (central) package may match a dependency; but with this mode
-    # enabled, a new local package of the same version will be installed as well.
+    # available, if possible. For example, if you are performing a local
+    # install, a released (central) package may match a dependency; but
+    # with this mode enabled, a new local package of the same version
+    # will be installed as well.
     #
-    # Typically, if performing a central install with the rez-pip --release flag,
-    # max_deps is equivalent to new_deps.
+    # Typically, if performing a central install with the
+    # rez-pip --release flag, max_deps is equivalent to new_deps.
     max_deps = 3
 
 
@@ -85,7 +85,7 @@ def is_exe(fpath):
         return os.path.exists(fpath) and os.access(fpath, os.X_OK)
 
 
-def run_pip_command(command_args, pip_version=None, python_version=None):
+def run_pip_command(command_args, python_version=None):
     """Run a pip command.
 
     Args:
@@ -94,16 +94,16 @@ def run_pip_command(command_args, pip_version=None, python_version=None):
     Returns:
         `subprocess.Popen`: Pip process.
     """
-    pip_exe, context = find_pip(pip_version, python_version)
-    command = [pip_exe] + list(command_args)
+    python_exe, context = find_python(python_version)
+    command = [python_exe, "-m", "pip"] + list(command_args)
 
     if context is None:
         return popen(command)
     else:
         return context.execute_shell(command=command, block=False)
 
 
-def find_pip(pip_version=None, python_version=None):
+def find_python(python_version=None):
     """Find a pip exe using the given python version.
 
     Returns:
@@ -112,44 +112,40 @@ def find_pip(pip_version=None, python_version=None):
             `ResolvedContext`: Context containing pip, or None if we fell back
                 to system pip.
     """
-    pip_exe = "pip"
+    python_exe = "python"
 
     try:
-        context = create_context(pip_version, python_version)
+        context = create_context(python_version)
     except BuildError as e:
         # fall back on system pip. Not ideal but at least it's something
         from rez.backport.shutilwhich import which
 
-        pip_exe = which("pip")
+        python_exe = which("python")
 
-        if pip_exe:
+        if python_exe:
             print_warning(
-                "pip rez package could not be found; system 'pip' command (%s) "
-                "will be used instead." % pip_exe)
+                "python rez package could not be found; system 'python' "
+                "command (%s) will be used instead." % python_exe)
             context = None
         else:
             raise e
 
-    return pip_exe, context
+    return python_exe, context
 
 
-def create_context(pip_version=None, python_version=None):
+def create_context(python_version=None):
     """Create a context containing the specific pip and python.
 
     Args:
-        pip_version (str or `Version`): Version of pip to use, or latest if None.
-        python_version (str or `Version`): Python version to use, or latest if
-            None.
+        python_version (str or `Version`): Python version to use,
+            or latest if None.
 
     Returns:
         `ResolvedContext`: Context containing pip and python.
+
     """
-    # determine pip pkg to use for install, and python variants to install on
-    if pip_version:
-        pip_req = "pip-%s" % str(pip_version)
-    else:
-        pip_req = "pip"
 
+    # determine pip pkg to use for install, and python variants to install on
     if python_version:
         ver = Version(str(python_version))
         major_minor_ver = ver.trim(2)
@@ -166,35 +162,35 @@ def create_context(pip_version=None, python_version=None):
 
         py_req = "python-%s" % str(major_minor_ver)
 
-    # use pip + latest python to perform pip download operations
-    request = [pip_req, py_req]
+    # use specified version of python to perform pip download operations
+    request = [py_req]
 
-    with convert_errors(from_=(PackageFamilyNotFoundError, PackageNotFoundError),
+    with convert_errors(from_=(PackageFamilyNotFoundError,
+                               PackageNotFoundError),
                         to=BuildError, msg="Cannot run - pip or python rez "
                         "package is not present"):
         context = ResolvedContext(request)
 
     # print pip package used to perform the install
-    pip_variant = context.get_resolved_package("pip")
-    pip_package = pip_variant.parent
-    print_info("Using %s (%s)" % (pip_package.qualified_name, pip_variant.uri))
+    python_variant = context.get_resolved_package("python")
+    python_package = python_variant.parent
+    print_info("Using %s (%s)" % (python_package.qualified_name,
+                                  python_variant.uri))
 
     return context
 
 
-def pip_install_package(source_name, pip_version=None, python_version=None,
+def pip_install_package(source_name, python_version=None,
                         mode=InstallMode.min_deps, release=False):
     """Install a pip-compatible python package as a rez package.
     Args:
         source_name (str): Name of package or archive/url containing the pip
             package source. This is the same as the arg you would pass to
             the 'pip install' command.
-        pip_version (str or `Version`): Version of pip to use to perform the
-            install, uses latest if None.
         python_version (str or `Version`): Python version to use to perform the
             install, and subsequently have the resulting rez package depend on.
-        mode (`InstallMode`): Installation mode, determines how dependencies are
-            managed.
+        mode (`InstallMode`): Installation mode, determines how
+            dependencies are managed.
         release (bool): If True, install as a released package; otherwise, it
             will be installed as a local package.
 
@@ -206,9 +202,10 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
     installed_variants = []
     skipped_variants = []
 
-    pip_exe, context = find_pip(pip_version, python_version)
+    python_exe, context = find_python(python_version)
 
-    # TODO: should check if packages_path is writable before continuing with pip
+    # TODO: should check if packages_path is writable
+    # before continuing with pip
     #
     packages_path = (config.release_packages_path if release
                      else config.local_packages_path)
@@ -218,9 +215,6 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
     stagingsep = "".join([os.path.sep, "rez_staging", os.path.sep])
 
     destpath = os.path.join(stagingdir, "python")
-    binpath = os.path.join(stagingdir, "bin")
-    incpath = os.path.join(stagingdir, "include")
-    datapath = stagingdir
 
     if context and config.debug("package_release"):
         buf = StringIO()
@@ -229,11 +223,10 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
         _log(buf.getvalue())
 
     # Build pip commandline
-    cmd = [pip_exe, "install",
-           "--install-option=--install-lib=%s" % destpath,
-           "--install-option=--install-scripts=%s" % binpath,
-           "--install-option=--install-headers=%s" % incpath,
-           "--install-option=--install-data=%s" % datapath]
+    cmd = [
+        python_exe, "-m", "pip", "install",
+        "--target", destpath
+    ]
 
     if mode == InstallMode.no_deps:
         cmd.append("--no-deps")
@@ -249,31 +242,39 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
     for distribution in distribution_path.get_distributions():
         requirements = []
         if distribution.metadata.run_requires:
-            # Handle requirements. Currently handles conditional environment based
+            # Handle requirements. Currently handles
+            # conditional environment based
             # requirements and normal requirements
             # TODO: Handle optional requirements?
             for requirement in distribution.metadata.run_requires:
                 if "environment" in requirement:
                     if interpret(requirement["environment"]):
-                        requirements.extend(_get_dependencies(requirement, distributions))
+                        requirements.extend(_get_dependencies(
+                            requirement, distributions))
                 elif "extra" in requirement:
                     # Currently ignoring optional requirements
                     pass
                 else:
-                    requirements.extend(_get_dependencies(requirement, distributions))
+                    requirements.extend(_get_dependencies(
+                        requirement, distributions))
 
         tools = []
         src_dst_lut = {}
+        files = distribution.list_installed_files()
 
-        for installed_file in distribution.list_installed_files(allow_fail=True):
-            source_file = os.path.normpath(os.path.join(destpath, installed_file[0]))
+        for installed_file in files:
+            source_file = os.path.join(destpath, installed_file[0])
+            source_file = os.path.normpath(source_file)
 
             if os.path.exists(source_file):
-                destination_file = installed_file[0].split(stagingsep)[1]
+                destination_file = source_file.split(stagingsep)[1]
                 exe = False
 
-                if is_exe(source_file) and \
-                        destination_file.startswith("%s%s" % ("bin", os.path.sep)):
+                starts_with_bin = destination_file.startswith(
+                    "%s%s" % ("bin", os.path.sep)
+                )
+
+                if is_exe(source_file) and starts_with_bin:
                     _, _file = os.path.split(destination_file)
                     tools.append(_file)
                     exe = True
@@ -285,12 +286,15 @@ def pip_install_package(source_name, pip_version=None, python_version=None,
 
         def make_root(variant, path):
             """Using distlib to iterate over all installed files of the current
-            distribution to copy files to the target directory of the rez package
-            variant
+            distribution to copy files to the target directory of the rez
+            package variant
+
             """
+
             for source_file, data in src_dst_lut.items():
                 destination_file, exe = data
-                destination_file = os.path.normpath(os.path.join(path, destination_file))
+                destination_file = os.path.join(path, destination_file)
+                destination_file = os.path.normpath(destination_file)
 
                 if not os.path.exists(os.path.dirname(destination_file)):
                     os.makedirs(os.path.dirname(destination_file))
@@ -307,7 +311,8 @@ def make_root(variant, path):
         variant_reqs.append("os-%s" % _system.os)
 
         if context is None:
-            # since we had to use system pip, we have to assume system python version
+            # since we had to use system pip, we have
+            # to assume system python version
             py_ver = '.'.join(map(str, sys.version_info[:2]))
         else:
             python_variant = context.get_resolved_package("python")

src/rez/vendor/distlib/__init__.py

@@ -1,12 +1,12 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2012-2016 Vinay Sajip.
+# Copyright (C) 2012-2017 Vinay Sajip.
 # Licensed to the Python Software Foundation under a contributor agreement.
 # See LICENSE.txt and CONTRIBUTORS.txt.
 #
 import logging
 
-__version__ = '0.2.4.dev0'
+__version__ = '0.2.8'
 
 class DistlibException(Exception):
     pass

src/rez/vendor/distlib/_backport/shutil.py

@@ -55,8 +55,8 @@ class ReadError(EnvironmentError):
     """Raised when an archive cannot be read"""
 
 class RegistryError(Exception):
-    """Raised when a registery operation with the archiving
-    and unpacking registeries fails"""
+    """Raised when a registry operation with the archiving
+    and unpacking registries fails"""
 
 
 try:
@@ -648,7 +648,7 @@ def register_unpack_format(name, extensions, function, extra_args=None,
     _UNPACK_FORMATS[name] = extensions, function, extra_args, description
 
 def unregister_unpack_format(name):
-    """Removes the pack format from the registery."""
+    """Removes the pack format from the registry."""
     del _UNPACK_FORMATS[name]
 
 def _ensure_directory(path):

src/rez/vendor/distlib/_backport/sysconfig.py

@@ -221,7 +221,7 @@ def _parse_makefile(filename, vars=None):
     """
     # Regexes needed for parsing Makefile (and similar syntaxes,
     # like old-style Setup files).
-    _variable_rx = re.compile("([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
+    _variable_rx = re.compile(r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)")
     _findvar1_rx = re.compile(r"\$\(([A-Za-z][A-Za-z0-9_]*)\)")
     _findvar2_rx = re.compile(r"\${([A-Za-z][A-Za-z0-9_]*)}")
 
@@ -537,7 +537,7 @@ def get_config_vars(*args):
                         # patched up as well.
                         'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
                     flags = _CONFIG_VARS[key]
-                    flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                    flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
                     flags = re.sub('-isysroot [^ \t]*', ' ', flags)
                     _CONFIG_VARS[key] = flags
             else:
@@ -554,7 +554,7 @@ def get_config_vars(*args):
                         'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
 
                         flags = _CONFIG_VARS[key]
-                        flags = re.sub('-arch\s+\w+\s', ' ', flags)
+                        flags = re.sub(r'-arch\s+\w+\s', ' ', flags)
                         flags = flags + ' ' + arch
                         _CONFIG_VARS[key] = flags
 
@@ -569,7 +569,7 @@ def get_config_vars(*args):
                 # when you install Xcode.
                 #
                 CFLAGS = _CONFIG_VARS.get('CFLAGS', '')
-                m = re.search('-isysroot\s+(\S+)', CFLAGS)
+                m = re.search(r'-isysroot\s+(\S+)', CFLAGS)
                 if m is not None:
                     sdk = m.group(1)
                     if not os.path.exists(sdk):
@@ -579,7 +579,7 @@ def get_config_vars(*args):
                             'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
 
                             flags = _CONFIG_VARS[key]
-                            flags = re.sub('-isysroot\s+\S+(\s|$)', ' ', flags)
+                            flags = re.sub(r'-isysroot\s+\S+(\s|$)', ' ', flags)
                             _CONFIG_VARS[key] = flags
 
     if args:
@@ -725,7 +725,7 @@ def get_platform():
                 machine = 'fat'
                 cflags = get_config_vars().get('CFLAGS')
 
-                archs = re.findall('-arch\s+(\S+)', cflags)
+                archs = re.findall(r'-arch\s+(\S+)', cflags)
                 archs = tuple(sorted(set(archs)))
 
                 if len(archs) == 1:

src/rez/vendor/distlib/_backport/tarfile.py

@@ -331,7 +331,7 @@ class ExtractError(TarError):
     """General exception for extract errors."""
     pass
 class ReadError(TarError):
-    """Exception for unreadble tar archives."""
+    """Exception for unreadable tar archives."""
     pass
 class CompressionError(TarError):
     """Exception for unavailable compression methods."""