Skip to content

Commit

Permalink
Escape venvs unless PEX_INHERIT_PATH is requested. (#1130)
Browse files Browse the repository at this point in the history 
Virtual environments created with --system-site-packages can break PEX
sys.path scrubbing. Avoid this class of environment leaks by breaking out
of venvs unless PEX_INHERIT_PATH has been requested.

A test is added that fails for virtualenv-16.7.10 without the fix.

Fixes #1031
  • Loading branch information
@jsirois
jsirois authored Dec 11, 2020
1 parent c726b43 commit 999f903
Show file tree
Hide file tree
Showing 4 changed files with 226 additions and 116 deletions.
14 changes: 11 additions & 3 deletions pex/interpreter_constraints.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@

from __future__ import absolute_import

import os

from pex.common import die
from pex.interpreter import PythonIdentity, PythonInterpreter
from pex.typing import TYPE_CHECKING
Expand Down Expand Up @@ -37,17 +35,27 @@ def __init__(
constraints, # type: Iterable[str]
candidates, # type: Iterable[PythonInterpreter]
failures, # type: Iterable[InterpreterIdentificationError]
preamble=None, # type: Optional[str]
):
# type: (...) -> None
"""
:param constraints: The constraints that could not be satisfied.
:param candidates: The python interpreters that were compared against the constraints.
:param failures: Descriptions of the python interpreters that were unidentifiable.
:param preamble: An optional preamble for the exception message.
"""
self.constraints = tuple(constraints)
self.candidates = tuple(candidates)
self.failures = tuple(failures)
super(UnsatisfiableInterpreterConstraintsError, self).__init__(self.create_message())
super(UnsatisfiableInterpreterConstraintsError, self).__init__(
self.create_message(preamble=preamble)
)

def with_preamble(self, preamble):
# type: (str) -> UnsatisfiableInterpreterConstraintsError
return UnsatisfiableInterpreterConstraintsError(
self.constraints, self.candidates, self.failures, preamble=preamble
)

def create_message(self, preamble=None):
# type: (Optional[str]) -> str
Expand Down
185 changes: 98 additions & 87 deletions pex/pex_bootstrapper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from pex import pex_warnings
from pex.common import die
from pex.inherit_path import InheritPath
from pex.interpreter import PythonInterpreter
from pex.interpreter_constraints import UnsatisfiableInterpreterConstraintsError
from pex.orderedset import OrderedSet
Expand Down Expand Up @@ -157,13 +158,13 @@ def _valid_interpreter(interp_or_error):
def _select_path_interpreter(
path=None, # type: Optional[str]
valid_basenames=None, # type: Optional[Tuple[str, ...]]
compatibility_constraints=None, # type: Optional[Iterable[str]]
interpreter_constraints=None, # type: Optional[Iterable[str]]
):
# type: (...) -> Optional[PythonInterpreter]
candidate_interpreters_iter = iter_compatible_interpreters(
path=path,
valid_basenames=valid_basenames,
interpreter_constraints=compatibility_constraints,
interpreter_constraints=interpreter_constraints,
)
current_interpreter = PythonInterpreter.get() # type: PythonInterpreter
candidate_interpreters = []
Expand All @@ -182,133 +183,143 @@ def _select_path_interpreter(
return PythonInterpreter.latest_release_of_min_compatible_version(candidate_interpreters)


def maybe_reexec_pex(compatibility_constraints=None):
# type: (Optional[Iterable[str]]) -> Union[None, NoReturn]
"""Handle environment overrides for the Python interpreter to use when executing this pex.
This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO
metadata. If PEX_PYTHON is set it attempts to obtain the binary location of the interpreter
specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for a matching
interpreter in accordance with the interpreter constraints. If both variables are present, this
function gives precedence to PEX_PYTHON_PATH and errors out if no compatible interpreters can be
found on said path.
If neither variable is set, we fall back to plain PEX execution using PATH searching or the
currently executing interpreter. If compatibility constraints are used, we match those constraints
against these interpreters.
:param compatibility_constraints: optional list of requirements-style strings that constrain the
Python interpreter to re-exec this pex with.
"""

def find_compatible_interpreter(interpreter_constraints=None):
# type: (Optional[Iterable[str]]) -> PythonInterpreter
current_interpreter = PythonInterpreter.get()
target = None # type: Optional[PythonInterpreter]

# NB: Used only for tests.
if "_PEX_EXEC_CHAIN" in os.environ:
flag_or_chain = os.environ.pop("_PEX_EXEC_CHAIN")
pex_exec_chain = [] if flag_or_chain == "1" else flag_or_chain.split(os.pathsep)
pex_exec_chain.append(current_interpreter.binary)
os.environ["_PEX_EXEC_CHAIN"] = os.pathsep.join(pex_exec_chain)

current_interpreter_blessed_env_var = "_PEX_SHOULD_EXIT_BOOTSTRAP_REEXEC"
if os.environ.pop(current_interpreter_blessed_env_var, None):
# We've already been here and selected an interpreter. Continue to execution.
return None

from . import pex

pythonpath = pex.PEX.stash_pythonpath()
if pythonpath is not None:
TRACER.log("Stashed PYTHONPATH of {}".format(pythonpath), V=2)

target = current_interpreter # type: Optional[PythonInterpreter]
with TRACER.timed("Selecting runtime interpreter", V=3):
if ENV.PEX_PYTHON and not ENV.PEX_PYTHON_PATH:
# preserve PEX_PYTHON re-exec for backwards compatibility
# TODO: Kill this off completely in favor of PEX_PYTHON_PATH
# https://github.com/pantsbuild/pex/issues/431
TRACER.log(
"Using PEX_PYTHON={} constrained by {}".format(
ENV.PEX_PYTHON, compatibility_constraints
ENV.PEX_PYTHON, interpreter_constraints
),
V=3,
)
try:
if os.path.isabs(ENV.PEX_PYTHON):
target = _select_path_interpreter(
path=ENV.PEX_PYTHON,
compatibility_constraints=compatibility_constraints,
interpreter_constraints=interpreter_constraints,
)
else:
target = _select_path_interpreter(
valid_basenames=(os.path.basename(ENV.PEX_PYTHON),),
compatibility_constraints=compatibility_constraints,
interpreter_constraints=interpreter_constraints,
)
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Failed to find a compatible PEX_PYTHON={pex_python}.".format(
pex_python=ENV.PEX_PYTHON
)
raise e.with_preamble(
"Failed to find a compatible PEX_PYTHON={pex_python}.".format(
pex_python=ENV.PEX_PYTHON
)
)
elif ENV.PEX_PYTHON_PATH or compatibility_constraints:
elif ENV.PEX_PYTHON_PATH or interpreter_constraints:
TRACER.log(
"Using {path} constrained by {constraints}".format(
path="PEX_PYTHON_PATH={}".format(ENV.PEX_PYTHON_PATH)
if ENV.PEX_PYTHON_PATH
else "$PATH",
constraints=compatibility_constraints,
constraints=interpreter_constraints,
),
V=3,
)
try:
target = _select_path_interpreter(
path=ENV.PEX_PYTHON_PATH, compatibility_constraints=compatibility_constraints
path=ENV.PEX_PYTHON_PATH, interpreter_constraints=interpreter_constraints
)
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Failed to find compatible interpreter on path {path}.".format(
path=ENV.PEX_PYTHON_PATH or os.getenv("PATH")
)
raise e.with_preamble(
"Failed to find compatible interpreter on path {path}.".format(
path=ENV.PEX_PYTHON_PATH or os.getenv("PATH")
)
)
elif pythonpath is None:
TRACER.log(
"Using the current interpreter {} since no constraints have been specified and "
"PYTHONPATH is not set.".format(sys.executable),
V=3,
)
return None
else:
target = current_interpreter

if not target:
# N.B.: This can only happen when PEX_PYTHON_PATH is set and compatibility_constraints is
# not set, but we handle all constraints generally for sanity sake.
constraints = []
if ENV.PEX_PYTHON:
constraints.append("PEX_PYTHON={}".format(ENV.PEX_PYTHON))
if ENV.PEX_PYTHON_PATH:
constraints.append("PEX_PYTHON_PATH={}".format(ENV.PEX_PYTHON_PATH))
if compatibility_constraints:
constraints.extend(
"--interpreter-constraint={}".format(compatibility_constraint)
for compatibility_constraint in compatibility_constraints

if target is None:
# N.B.: This can only happen when PEX_PYTHON_PATH is set and interpreter_constraints
# is empty, but we handle all constraints generally for sanity sake.
constraints = []
if ENV.PEX_PYTHON:
constraints.append("PEX_PYTHON={}".format(ENV.PEX_PYTHON))
if ENV.PEX_PYTHON_PATH:
constraints.append("PEX_PYTHON_PATH={}".format(ENV.PEX_PYTHON_PATH))
if interpreter_constraints:
constraints.append(
"Version matches {}".format(" or ".join(interpreter_constraints))
)
raise UnsatisfiableInterpreterConstraintsError(
constraints=constraints,
candidates=[current_interpreter],
failures=[],
preamble="Could not find a compatible interpreter.",
)
return target

die(
"Failed to find an appropriate Python interpreter.\n"
"\n"
"Although the current interpreter is {python}, the following constraints exclude it:\n"
" {constraints}".format(python=sys.executable, constraints="\n ".join(constraints))

def maybe_reexec_pex(interpreter_constraints=None):
# type: (Optional[Iterable[str]]) -> Union[None, NoReturn]
"""Handle environment overrides for the Python interpreter to use when executing this pex.
This function supports interpreter filtering based on interpreter constraints stored in PEX-INFO
metadata. If PEX_PYTHON is set it attempts to obtain the binary location of the interpreter
specified by PEX_PYTHON. If PEX_PYTHON_PATH is set, it attempts to search the path for a
matching interpreter in accordance with the interpreter constraints. If both variables are
present, this function gives precedence to PEX_PYTHON_PATH and errors out if no compatible
interpreters can be found on said path.
If neither variable is set, we fall back to plain PEX execution using PATH searching or the
currently executing interpreter. If compatibility constraints are used, we match those
constraints against these interpreters.
:param interpreter_constraints: Optional list of requirements-style strings that constrain the
Python interpreter to re-exec this pex with.
"""

current_interpreter = PythonInterpreter.get()

# NB: Used only for tests.
if "_PEX_EXEC_CHAIN" in os.environ:
flag_or_chain = os.environ.pop("_PEX_EXEC_CHAIN")
pex_exec_chain = [] if flag_or_chain == "1" else flag_or_chain.split(os.pathsep)
pex_exec_chain.append(current_interpreter.binary)
os.environ["_PEX_EXEC_CHAIN"] = os.pathsep.join(pex_exec_chain)

current_interpreter_blessed_env_var = "_PEX_SHOULD_EXIT_BOOTSTRAP_REEXEC"
if os.environ.pop(current_interpreter_blessed_env_var, None):
# We've already been here and selected an interpreter. Continue to execution.
return None

try:
target = find_compatible_interpreter(
interpreter_constraints=interpreter_constraints,
)
except UnsatisfiableInterpreterConstraintsError as e:
die(str(e))

os.environ.pop("PEX_PYTHON", None)
os.environ.pop("PEX_PYTHON_PATH", None)

if pythonpath is None and target == current_interpreter:
if ENV.PEX_INHERIT_PATH == InheritPath.FALSE:
# Now that we've found a compatible Python interpreter, make sure we resolve out of any
# virtual environments it may be contained in since virtual environments created with
# `--system-site-packages` foil PEX attempts to scrub the sys.path.
resolved = target.resolve_base_interpreter()
if resolved != target:
TRACER.log(
"Resolved base interpreter of {} from virtual environment at {}".format(
resolved, target.prefix
),
V=3,
)
target = resolved

from . import pex

pythonpath = pex.PEX.stash_pythonpath()
if pythonpath is not None:
TRACER.log("Stashed PYTHONPATH of {}".format(pythonpath), V=2)
elif target == current_interpreter:
TRACER.log(
"Using the current interpreter {} since it matches constraints and "
"PYTHONPATH is not set.".format(sys.executable)
Expand All @@ -323,13 +334,13 @@ def maybe_reexec_pex(compatibility_constraints=None):
"sys.executable={python!r}, "
"PEX_PYTHON={pex_python!r}, "
"PEX_PYTHON_PATH={pex_python_path!r}, "
"COMPATIBILITY_CONSTRAINTS={compatibility_constraints!r}"
"interpreter_constraints={interpreter_constraints!r}"
"{pythonpath}".format(
cmdline=" ".join(cmdline),
python=sys.executable,
pex_python=ENV.PEX_PYTHON,
pex_python_path=ENV.PEX_PYTHON_PATH,
compatibility_constraints=compatibility_constraints,
interpreter_constraints=interpreter_constraints,
pythonpath=', (stashed) PYTHONPATH="{}"'.format(pythonpath)
if pythonpath is not None
else "",
Expand Down
16 changes: 11 additions & 5 deletions pex/testing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -479,17 +479,23 @@ def run_pyenv(args):
return python, pip, run_pyenv


def ensure_python_venv(version, latest_pip=True):
def ensure_python_venv(version, latest_pip=True, system_site_packages=False):
python, pip, _ = ensure_python_distribution(version)
venv = safe_mkdtemp()
if version in _ALL_PY3_VERSIONS:
subprocess.check_call([python, "-m", "venv", venv])
args = [python, "-m", "venv", venv]
if system_site_packages:
args.append("--system-site-packages")
subprocess.check_call(args=args)
else:
subprocess.check_call([pip, "install", "virtualenv==16.7.10"])
subprocess.check_call([python, "-m", "virtualenv", venv])
subprocess.check_call(args=[pip, "install", "virtualenv==16.7.10"])
args = [python, "-m", "virtualenv", venv, "-q"]
if system_site_packages:
args.append("--system-site-packages")
subprocess.check_call(args=args)
python, pip = tuple(os.path.join(venv, "bin", exe) for exe in ("python", "pip"))
if latest_pip:
subprocess.check_call([pip, "install", "-U", "pip"])
subprocess.check_call(args=[pip, "install", "-U", "pip"])
return python, pip


Expand Down
Loading

0 comments on commit 999f903

Please sign in to comment.