From 75a69f9c799d0ad0f4393cd06eab92db4f454931 Mon Sep 17 00:00:00 2001 From: Josh Snyder Date: Mon, 25 Apr 2022 12:33:34 -0700 Subject: [PATCH] reproducible directories for pip builds Currently, pip randomly assigns directory names when it builds Python sdists into bdists. This can result in randomized file paths being embedded into the build output (usually in debug symbols, but potentially in other places). The ideal solution would be to trim the front (random part) of the file path off, leaving the remaining (deterministic) part to embed in the binary. Doing so would require reaching deep into the configuration of whatever compiler/linker pip happens to be using (e.g. gcc, clang, rustc, etc.). This option, on the other hand, doesn't require modifying the internals of Python packages. In this patch we make it so that pip's randomly assigned directory paths are instead generated from a deterministic counter. Doing so requires exclusive access to TMPDIR, because otherwise other programs (likely other executions of `pip`) will attempt to create directories of the same name. For that reason, the feature only activates when SOURCE_DATE_EPOCH is set. For more discussion (and prior art) in this area, see: * https://github.com/NixOS/nixpkgs/pull/102222/files * https://github.com/pypa/pip/issues/6505 --- src/pip/_internal/build_env.py | 11 +++++++++- .../operations/build/metadata_legacy.py | 11 ++++++++-- .../operations/build/wheel_legacy.py | 13 +++++++++++- src/pip/_internal/utils/temp_dir.py | 21 +++++++++++++++++++ src/pip/_internal/wheel_builder.py | 2 +- 5 files changed, 53 insertions(+), 5 deletions(-) diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index cdf04324107..9dcd49740b2 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -70,9 +70,11 @@ class BuildEnvironment: def __init__(self): # type: () -> None + temp_dir = TempDirectory( kind=tempdir_kinds.BUILD_ENV, globally_managed=True ) + self._sub_temp_dir = temp_dir.make_sub_temp_dir() self._prefixes = OrderedDict( (name, _Prefix(os.path.join(temp_dir.path, name))) @@ -126,7 +128,7 @@ def __enter__(self): # type: () -> None self._save_env = { name: os.environ.get(name, None) - for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') + for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH', 'TMPDIR') } path = self._bin_dirs[:] @@ -141,6 +143,8 @@ def __enter__(self): 'PYTHONNOUSERSITE': '1', 'PYTHONPATH': os.pathsep.join(pythonpath), }) + if self._sub_temp_dir is not None: + os.environ['TMPDIR'] = self._sub_temp_dir def __exit__( self, @@ -202,6 +206,7 @@ def install_requirements( requirements, prefix, message, + self._sub_temp_dir, ) @staticmethod @@ -211,6 +216,7 @@ def _install_requirements( requirements: Iterable[str], prefix: _Prefix, message: str, + sub_temp_dir: str, ) -> None: args = [ sys.executable, pip_runnable, 'install', @@ -243,6 +249,9 @@ def _install_requirements( args.append('--') args.extend(requirements) extra_environ = {"_PIP_STANDALONE_CERT": where()} + if sub_temp_dir is not None: + extra_environ["TMPDIR"] = sub_temp_dir + with open_spinner(message) as spinner: call_subprocess(args, spinner=spinner, extra_environ=extra_environ) diff --git a/src/pip/_internal/operations/build/metadata_legacy.py b/src/pip/_internal/operations/build/metadata_legacy.py index f46538a07f4..95cb4b72a7f 100644 --- a/src/pip/_internal/operations/build/metadata_legacy.py +++ b/src/pip/_internal/operations/build/metadata_legacy.py @@ -53,9 +53,10 @@ def generate_metadata( setup_py_path, details, ) - egg_info_dir = TempDirectory( + tmp_dir = TempDirectory( kind="pip-egg-info", globally_managed=True - ).path + ) + egg_info_dir = tmp_dir.path args = make_setuptools_egg_info_args( setup_py_path, @@ -63,10 +64,16 @@ def generate_metadata( no_user_config=isolated, ) + extra_environ = dict() + sub_temp_dir = tmp_dir.make_sub_temp_dir() + if sub_temp_dir is not None: + extra_environ["TMPDIR"] = sub_temp_dir, + with build_env: call_subprocess( args, cwd=source_dir, + extra_environ=extra_environ, command_desc='python setup.py egg_info', ) diff --git a/src/pip/_internal/operations/build/wheel_legacy.py b/src/pip/_internal/operations/build/wheel_legacy.py index 755c3bc83a2..559c2ae6030 100644 --- a/src/pip/_internal/operations/build/wheel_legacy.py +++ b/src/pip/_internal/operations/build/wheel_legacy.py @@ -1,5 +1,6 @@ import logging import os.path +import shutil from typing import List, Optional from pip._internal.cli.spinners import open_spinner @@ -9,6 +10,7 @@ call_subprocess, format_command_args, ) +from pip._internal.utils.temp_dir import TempDirectory logger = logging.getLogger(__name__) @@ -70,13 +72,14 @@ def build_wheel_legacy( source_dir, # type: str global_options, # type: List[str] build_options, # type: List[str] - tempd, # type: str + temp_dir, # type: TempDirectory ): # type: (...) -> Optional[str] """Build one unpacked package using the "legacy" build process. Returns path to wheel if successfully built. Otherwise, returns None. """ + tempd = temp_dir.path wheel_args = make_setuptools_bdist_wheel_args( setup_py_path, global_options=global_options, @@ -88,10 +91,16 @@ def build_wheel_legacy( with open_spinner(spin_message) as spinner: logger.debug('Destination directory: %s', tempd) + sub_temp_dir = temp_dir.make_sub_temp_dir() + extra_environ = dict() + if sub_temp_dir is not None: + extra_environ["TMPDIR"] = sub_temp_dir + try: output = call_subprocess( wheel_args, cwd=source_dir, + extra_environ=extra_environ, spinner=spinner, ) except Exception: @@ -99,6 +108,8 @@ def build_wheel_legacy( logger.error('Failed building wheel for %s', name) return None + shutil.rmtree(sub_temp_dir) + names = os.listdir(tempd) wheel_path = get_legacy_build_wheel_path( names=names, diff --git a/src/pip/_internal/utils/temp_dir.py b/src/pip/_internal/utils/temp_dir.py index 477cbe6b1aa..c737866f1f7 100644 --- a/src/pip/_internal/utils/temp_dir.py +++ b/src/pip/_internal/utils/temp_dir.py @@ -1,6 +1,7 @@ import errno import itertools import logging +import os import os.path import tempfile from contextlib import ExitStack, contextmanager @@ -12,6 +13,7 @@ _T = TypeVar("_T", bound="TempDirectory") +tmpdir_serial = 0 # Kinds of temporary directories. Only needed for ones that are # globally-managed. @@ -171,6 +173,17 @@ def _create(self, kind): # symlinked to another directory. This tends to confuse build # scripts, so we canonicalize the path by traversing potential # symlinks here. + + if "SOURCE_DATE_EPOCH" in os.environ: + global tmpdir_serial + path = os.path.join( + tempfile.gettempdir(), + "pip-{}-{}".format(kind, tmpdir_serial) + ) + tmpdir_serial += 1 + os.mkdir(path) + return path + path = os.path.realpath(tempfile.mkdtemp(prefix=f"pip-{kind}-")) logger.debug("Created temporary directory: %s", path) return path @@ -183,6 +196,14 @@ def cleanup(self): return rmtree(self._path) + def make_sub_temp_dir(self): + if "SOURCE_DATE_EPOCH" not in os.environ: + return None + + ret = os.path.join(self._path, 'tmp') + os.mkdir(ret) + return ret + class AdjacentTempDirectory(TempDirectory): """Helper class that creates a temporary directory adjacent to a real one. diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index 92f172bca16..0df0cd8812f 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -265,7 +265,7 @@ def _build_one_inside_env( source_dir=req.unpacked_source_directory, global_options=global_options, build_options=build_options, - tempd=temp_dir.path, + temp_dir=temp_dir, ) if wheel_path is not None: