Isolate the Pip cache per Pip version. (#2164)

Fixes #2163

pex/pip/installation.py

@@ -11,7 +11,7 @@
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
 from pex.pex import PEX
-from pex.pex_bootstrapper import VenvPex, ensure_venv
+from pex.pex_bootstrapper import ensure_venv
 from pex.pip.tool import Pip
 from pex.pip.version import PipVersion, PipVersionValue
 from pex.resolve.resolvers import Resolver
@@ -29,13 +29,14 @@
     from pex.third_party import attr
 
 
-def _pip_venv(
+def _pip_installation(
     version,  # type: PipVersionValue
     iter_distribution_locations,  # type: Callable[[], Iterator[str]]
     interpreter=None,  # type: Optional[PythonInterpreter]
 ):
-    # type: (...) -> VenvPex
-    path = os.path.join(ENV.PEX_ROOT, "pip-{version}.pex".format(version=version))
+    # type: (...) -> Pip
+    pip_root = os.path.join(ENV.PEX_ROOT, "pip", str(version))
+    path = os.path.join(pip_root, "pip.pex")
     pip_interpreter = interpreter or PythonInterpreter.get()
     pip_pex_path = os.path.join(path, isolated().pex_hash)
     with atomic_directory(pip_pex_path) as chroot:
@@ -66,13 +67,15 @@ def _pip_venv(
                 fp.close()
                 isolated_pip_builder.set_executable(fp.name, "__pex_patched_pip__.py")
             isolated_pip_builder.freeze()
-    return ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
+    pip_cache = os.path.join(pip_root, "pip_cache")
+    pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
+    return Pip(pip_pex=pip_pex, pip_cache=pip_cache)
 
 
 def _vendored_installation(interpreter=None):
-    # type: (Optional[PythonInterpreter]) -> VenvPex
+    # type: (Optional[PythonInterpreter]) -> Pip
 
-    return _pip_venv(
+    return _pip_installation(
         version=PipVersion.VENDORED,
         iter_distribution_locations=lambda: third_party.expose(("pip", "setuptools", "wheel")),
         interpreter=interpreter,
@@ -84,7 +87,7 @@ def _resolved_installation(
     resolver,  # type: Resolver
     interpreter=None,  # type: Optional[PythonInterpreter]
 ):
-    # type: (...) -> VenvPex
+    # type: (...) -> Pip
     if version is PipVersion.VENDORED:
         return _vendored_installation(interpreter=interpreter)
 
@@ -96,7 +99,7 @@ def resolve_distribution_locations():
         ).installed_distributions:
             yield installed_distribution.distribution.location
 
-    return _pip_venv(
+    return _pip_installation(
         version=version,
         iter_distribution_locations=resolve_distribution_locations,
         interpreter=interpreter,
@@ -204,20 +207,16 @@ def get_pip(
     if pip is None:
         installation.check_python_applies()
         if version is PipVersion.VENDORED:
-            pip = Pip(pip_pex=_vendored_installation(interpreter=interpreter))
+            pip = _vendored_installation(interpreter=interpreter)
         else:
             if resolver is None:
                 raise ValueError(
                     "A resolver is required to install {requirement}".format(
                         requirement=version.requirement
                     )
                 )
-            pip = Pip(
-                pip_pex=_resolved_installation(
-                    version=version,
-                    resolver=resolver,
-                    interpreter=interpreter,
-                )
+            pip = _resolved_installation(
+                version=version, resolver=resolver, interpreter=interpreter
             )
         _PIP[installation] = pip
     return pip

pex/pip/tool.py

@@ -224,6 +224,7 @@ class Pip(object):
     _PATCHES_PACKAGE_NAME = "_pex_pip_patches"
 
     _pip_pex = attr.ib()  # type: VenvPex
+    _pip_cache = attr.ib()  # type: str
 
     @staticmethod
     def _calculate_resolver_version(package_index_configuration=None):
@@ -301,8 +302,7 @@ def _spawn_pip_isolated(
         else:
             pip_args.append("-q")
 
-        pip_cache = os.path.join(ENV.PEX_ROOT, "pip_cache")
-        pip_args.extend(["--cache-dir", pip_cache])
+        pip_args.extend(["--cache-dir", self._pip_cache])
 
         command = pip_args + list(args)
 
@@ -321,7 +321,7 @@ def _spawn_pip_isolated(
         # since Pip relies upon `shutil.move` which is only atomic when `os.rename` can be used.
         # See https://github.com/pantsbuild/pex/issues/1776 for an example of the issues non-atomic
         # moves lead to in the `pip wheel` case.
-        pip_tmpdir = os.path.join(pip_cache, ".tmp")
+        pip_tmpdir = os.path.join(self._pip_cache, ".tmp")
         safe_mkdir(pip_tmpdir)
         extra_env.update(TMPDIR=pip_tmpdir)