reproducible directories for pip builds

Currently, pip randomly assigns directory names when it builds Python sdists
into bdists. This can result in randomized file paths being embedded into the
build output (usually in debug symbols, but potentially in other places). The
ideal solution would be to trim the front (random part) of the file path off,
leaving the remaining (deterministic) part to embed in the binary. Doing so
would require reaching deep into the configuration of whatever compiler/linker
pip happens to be using (e.g.  gcc, clang, rustc, etc.). This option, on the
other hand, doesn't require modifying the internals of Python packages.

In this patch we make it so that pip's randomly assigned directory paths are
instead generated from a deterministic counter. Doing so requires exclusive
access to TMPDIR, because otherwise other programs (likely other executions of
`pip`) will attempt to create directories of the same name. For that reason,
the feature only activates when SOURCE_DATE_EPOCH is set.

For more discussion (and prior art) in this area, see:
 * https://github.com/NixOS/nixpkgs/pull/102222/files
 * pypa#6505

src/pip/_internal/build_env.py

@@ -70,9 +70,11 @@ class BuildEnvironment:
 
     def __init__(self):
         # type: () -> None
+
         temp_dir = TempDirectory(
             kind=tempdir_kinds.BUILD_ENV, globally_managed=True
         )
+        self._sub_temp_dir = temp_dir.make_sub_temp_dir()
 
         self._prefixes = OrderedDict(
             (name, _Prefix(os.path.join(temp_dir.path, name)))
@@ -126,7 +128,7 @@ def __enter__(self):
         # type: () -> None
         self._save_env = {
             name: os.environ.get(name, None)
-            for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH')
+            for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH', 'TMPDIR')
         }
 
         path = self._bin_dirs[:]
@@ -141,6 +143,8 @@ def __enter__(self):
             'PYTHONNOUSERSITE': '1',
             'PYTHONPATH': os.pathsep.join(pythonpath),
         })
+        if self._sub_temp_dir is not None:
+            os.environ['TMPDIR'] = self._sub_temp_dir
 
     def __exit__(
         self,
@@ -202,6 +206,7 @@ def install_requirements(
                 requirements,
                 prefix,
                 message,
+                self._sub_temp_dir,
             )
 
     @staticmethod
@@ -211,6 +216,7 @@ def _install_requirements(
         requirements: Iterable[str],
         prefix: _Prefix,
         message: str,
+        sub_temp_dir: str,
     ) -> None:
         args = [
             sys.executable, pip_runnable, 'install',
@@ -243,6 +249,9 @@ def _install_requirements(
         args.append('--')
         args.extend(requirements)
         extra_environ = {"_PIP_STANDALONE_CERT": where()}
+        if sub_temp_dir is not None:
+            extra_environ["TMPDIR"] = sub_temp_dir
+
         with open_spinner(message) as spinner:
             call_subprocess(args, spinner=spinner, extra_environ=extra_environ)
 

src/pip/_internal/operations/build/metadata_legacy.py

@@ -53,20 +53,27 @@ def generate_metadata(
         setup_py_path, details,
     )
 
-    egg_info_dir = TempDirectory(
+    tmp_dir = TempDirectory(
         kind="pip-egg-info", globally_managed=True
-    ).path
+    )
+    egg_info_dir = tmp_dir.path
 
     args = make_setuptools_egg_info_args(
         setup_py_path,
         egg_info_dir=egg_info_dir,
         no_user_config=isolated,
     )
 
+    extra_environ = dict()
+    sub_temp_dir = tmp_dir.make_sub_temp_dir()
+    if sub_temp_dir is not None:
+        extra_environ["TMPDIR"] = sub_temp_dir
+
     with build_env:
         call_subprocess(
             args,
             cwd=source_dir,
+            extra_environ=extra_environ,
             command_desc='python setup.py egg_info',
         )
 

src/pip/_internal/operations/build/wheel_legacy.py

@@ -1,5 +1,6 @@
 import logging
 import os.path
+import shutil
 from typing import List, Optional
 
 from pip._internal.cli.spinners import open_spinner
@@ -9,6 +10,7 @@
     call_subprocess,
     format_command_args,
 )
+from pip._internal.utils.temp_dir import TempDirectory
 
 logger = logging.getLogger(__name__)
 
@@ -70,13 +72,14 @@ def build_wheel_legacy(
     source_dir,  # type: str
     global_options,  # type: List[str]
     build_options,  # type: List[str]
-    tempd,  # type: str
+    temp_dir,  # type: TempDirectory
 ):
     # type: (...) -> Optional[str]
     """Build one unpacked package using the "legacy" build process.
 
     Returns path to wheel if successfully built. Otherwise, returns None.
     """
+    tempd = temp_dir.path
     wheel_args = make_setuptools_bdist_wheel_args(
         setup_py_path,
         global_options=global_options,
@@ -88,17 +91,26 @@ def build_wheel_legacy(
     with open_spinner(spin_message) as spinner:
         logger.debug('Destination directory: %s', tempd)
 
+        sub_temp_dir = temp_dir.make_sub_temp_dir()
+        extra_environ = dict()
+        if sub_temp_dir is not None:
+            extra_environ["TMPDIR"] = sub_temp_dir
+
         try:
             output = call_subprocess(
                 wheel_args,
                 cwd=source_dir,
+                extra_environ=extra_environ,
                 spinner=spinner,
             )
         except Exception:
             spinner.finish("error")
             logger.error('Failed building wheel for %s', name)
             return None
 
+        if sub_temp_dir is not None:
+            shutil.rmtree(sub_temp_dir)
+
         names = os.listdir(tempd)
         wheel_path = get_legacy_build_wheel_path(
             names=names,

src/pip/_internal/utils/temp_dir.py

@@ -1,6 +1,7 @@
 import errno
 import itertools
 import logging
+import os
 import os.path
 import tempfile
 from contextlib import ExitStack, contextmanager
@@ -12,6 +13,7 @@
 
 _T = TypeVar("_T", bound="TempDirectory")
 
+tmpdir_serial = 0
 
 # Kinds of temporary directories. Only needed for ones that are
 # globally-managed.
@@ -171,6 +173,17 @@ def _create(self, kind):
         # symlinked to another directory.  This tends to confuse build
         # scripts, so we canonicalize the path by traversing potential
         # symlinks here.
+
+        if "SOURCE_DATE_EPOCH" in os.environ:
+            global tmpdir_serial
+            path = os.path.join(
+                tempfile.gettempdir(),
+                "pip-{}-{}".format(kind, tmpdir_serial)
+            )
+            tmpdir_serial += 1
+            os.mkdir(path)
+            return path
+
         path = os.path.realpath(tempfile.mkdtemp(prefix=f"pip-{kind}-"))
         logger.debug("Created temporary directory: %s", path)
         return path
@@ -183,6 +196,14 @@ def cleanup(self):
             return
         rmtree(self._path)
 
+    def make_sub_temp_dir(self):
+        if "SOURCE_DATE_EPOCH" not in os.environ:
+            return None
+
+        ret = os.path.join(self._path, 'tmp')
+        os.mkdir(ret)
+        return ret
+
 
 class AdjacentTempDirectory(TempDirectory):
     """Helper class that creates a temporary directory adjacent to a real one.

src/pip/_internal/wheel_builder.py

@@ -265,7 +265,7 @@ def _build_one_inside_env(
                 source_dir=req.unpacked_source_directory,
                 global_options=global_options,
                 build_options=build_options,
-                tempd=temp_dir.path,
+                temp_dir=temp_dir,
             )
 
         if wheel_path is not None: