Support PEP 405 include-system-site-packages configuration (#7155)

news/5702.bugfix

@@ -0,0 +1 @@
+Correctly handle system site-packages, in virtual environments created with venv (PEP 405).

news/7155.bugfix

@@ -0,0 +1 @@
+Correctly handle system site-packages, in virtual environments created with venv (PEP 405).

src/pip/_internal/utils/virtualenv.py

@@ -1,34 +1,115 @@
-import os.path
+from __future__ import absolute_import
+
+import logging
+import os
+import re
 import site
 import sys
 
+from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+
+if MYPY_CHECK_RUNNING:
+    from typing import List, Optional
+
+logger = logging.getLogger(__name__)
+_INCLUDE_SYSTEM_SITE_PACKAGES_REGEX = re.compile(
+    r"include-system-site-packages\s*=\s*(?P<value>true|false)"
+)
+
+
+def _running_under_venv():
+    # type: () -> bool
+    """Checks if sys.base_prefix and sys.prefix match.
+
+    This handles PEP 405 compliant virtual environments.
+    """
+    return sys.prefix != getattr(sys, "base_prefix", sys.prefix)
+
+
+def _running_under_regular_virtualenv():
+    # type: () -> bool
+    """Checks if sys.real_prefix is set.
+
+    This handles virtual environments created with pypa's virtualenv.
+    """
+    # pypa/virtualenv case
+    return hasattr(sys, 'real_prefix')
+
 
 def running_under_virtualenv():
     # type: () -> bool
+    """Return True if we're running inside a virtualenv, False otherwise.
     """
-    Return True if we're running inside a virtualenv, False otherwise.
+    return _running_under_venv() or _running_under_regular_virtualenv()
+
 
+def _get_pyvenv_cfg_lines():
+    # type: () -> Optional[List[str]]
+    """Reads {sys.prefix}/pyvenv.cfg and returns its contents as list of lines
+
+    Returns None, if it could not read/access the file.
     """
-    if hasattr(sys, 'real_prefix'):
-        # pypa/virtualenv case
-        return True
-    elif sys.prefix != getattr(sys, "base_prefix", sys.prefix):
-        # PEP 405 venv
+    pyvenv_cfg_file = os.path.join(sys.prefix, 'pyvenv.cfg')
+    try:
+        with open(pyvenv_cfg_file) as f:
+            return f.read().splitlines()  # avoids trailing newlines
+    except IOError:
+        return None
+
+
+def _no_global_under_venv():
+    # type: () -> bool
+    """Check `{sys.prefix}/pyvenv.cfg` for system site-packages inclusion
+
+    PEP 405 specifies that when system site-packages are not supposed to be
+    visible from a virtual environment, `pyvenv.cfg` must contain the following
+    line:
+
+        include-system-site-packages = false
+
+    Additionally, log a warning if accessing the file fails.
+    """
+    cfg_lines = _get_pyvenv_cfg_lines()
+    if cfg_lines is None:
+        # We're not in a "sane" venv, so assume there is no system
+        # site-packages access (since that's PEP 405's default state).
+        logger.warning(
+            "Could not access 'pyvenv.cfg' despite a virtual environment "
+            "being active. Assuming global site-packages is not accessible "
+            "in this environment."
+        )
         return True
 
+    for line in cfg_lines:
+        match = _INCLUDE_SYSTEM_SITE_PACKAGES_REGEX.match(line)
+        if match is not None and match.group('value') == 'false':
+            return True
     return False
 
 
-def virtualenv_no_global():
+def _no_global_under_regular_virtualenv():
     # type: () -> bool
+    """Check if "no-global-site-packages.txt" exists beside site.py
+
+    This mirrors logic in pypa/virtualenv for determining whether system
+    site-packages are visible in the virtual environment.
     """
-    Return True if in a venv and no system site packages.
-    """
-    # this mirrors the logic in virtualenv.py for locating the
-    # no-global-site-packages.txt file
     site_mod_dir = os.path.dirname(os.path.abspath(site.__file__))
-    no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt')
-    if running_under_virtualenv() and os.path.isfile(no_global_file):
-        return True
-    else:
-        return False
+    no_global_site_packages_file = os.path.join(
+        site_mod_dir, 'no-global-site-packages.txt',
+    )
+    return os.path.exists(no_global_site_packages_file)
+
+
+def virtualenv_no_global():
+    # type: () -> bool
+    """Returns a boolean, whether running in venv with no system site-packages.
+    """
+
+    if _running_under_regular_virtualenv():
+        return _no_global_under_regular_virtualenv()
+
+    if _running_under_venv():
+        return _no_global_under_venv()
+
+    return False

tests/functional/test_freeze.py

@@ -703,6 +703,7 @@ def test_freeze_with_requirement_option_package_repeated_multi_file(script):
 
 
 @pytest.mark.network
+@pytest.mark.incompatible_with_test_venv
 def test_freeze_user(script, virtualenv, data):
     """
     Testing freeze with --user, first we have to install some stuff.
@@ -733,6 +734,7 @@ def test_freeze_path(tmpdir, script, data):
     _check_output(result.stdout, expected)
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_freeze_path_exclude_user(tmpdir, script, data):
     """
     Test freeze with --path and make sure packages from --user are not picked

tests/functional/test_install.py

@@ -124,6 +124,7 @@ def test_pep518_allows_missing_requires(script, data, common_wheels):
     assert result.files_created
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_pep518_with_user_pip(script, pip_src, data, common_wheels):
     """
     Check that build dependencies are installed into the build
@@ -1593,6 +1594,7 @@ def test_target_install_ignores_distutils_config_install_prefix(script):
     assert relative_script_base not in result.files_created
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_user_config_accepted(script):
     # user set in the config file is parsed as 0/1 instead of True/False.
     # Check that this doesn't cause a problem.

tests/functional/test_install_reqs.py

@@ -227,6 +227,7 @@ def test_install_local_with_subdirectory(script):
     result.assert_installed('version_subpkg.py', editable=False)
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_wheel_user_with_prefix_in_pydistutils_cfg(
         script, data, with_wheel):
     if os.name == 'posix':

tests/functional/test_install_user.py

@@ -27,6 +27,7 @@ def dist_in_site_packages(dist):
 class Tests_UserSite:
 
     @pytest.mark.network
+    @pytest.mark.incompatible_with_test_venv
     def test_reset_env_system_site_packages_usersite(self, script):
         """
         Check user site works as expected.
@@ -42,6 +43,7 @@ def test_reset_env_system_site_packages_usersite(self, script):
 
     @pytest.mark.network
     @need_svn
+    @pytest.mark.incompatible_with_test_venv
     def test_install_subversion_usersite_editable_with_distribute(
             self, script, tmpdir):
         """
@@ -55,6 +57,7 @@ def test_install_subversion_usersite_editable_with_distribute(
         )
         result.assert_installed('INITools', use_user_site=True)
 
+    @pytest.mark.incompatible_with_test_venv
     def test_install_from_current_directory_into_usersite(
             self, script, data, with_wheel):
         """
@@ -75,7 +78,6 @@ def test_install_from_current_directory_into_usersite(
         )
         assert dist_info_folder in result.files_created
 
-    @pytest.mark.incompatible_with_test_venv
     def test_install_user_venv_nositepkgs_fails(self, virtualenv,
                                                 script, data):
         """
@@ -96,6 +98,7 @@ def test_install_user_venv_nositepkgs_fails(self, virtualenv,
         )
 
     @pytest.mark.network
+    @pytest.mark.incompatible_with_test_venv
     def test_install_user_conflict_in_usersite(self, script):
         """
         Test user install with conflict in usersite updates usersite.
@@ -119,6 +122,7 @@ def test_install_user_conflict_in_usersite(self, script):
         assert not isfile(initools_v3_file), initools_v3_file
 
     @pytest.mark.network
+    @pytest.mark.incompatible_with_test_venv
     def test_install_user_conflict_in_globalsite(self, virtualenv, script):
         """
         Test user install with conflict in global site ignores site and
@@ -149,6 +153,7 @@ def test_install_user_conflict_in_globalsite(self, virtualenv, script):
         assert isdir(initools_folder)
 
     @pytest.mark.network
+    @pytest.mark.incompatible_with_test_venv
     def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script):
         """
         Test user install/upgrade with conflict in global site ignores site and
@@ -178,6 +183,7 @@ def test_upgrade_user_conflict_in_globalsite(self, virtualenv, script):
         assert isdir(initools_folder)
 
     @pytest.mark.network
+    @pytest.mark.incompatible_with_test_venv
     def test_install_user_conflict_in_globalsite_and_usersite(
             self, virtualenv, script):
         """
@@ -214,6 +220,7 @@ def test_install_user_conflict_in_globalsite_and_usersite(
         assert isdir(initools_folder)
 
     @pytest.mark.network
+    @pytest.mark.incompatible_with_test_venv
     def test_install_user_in_global_virtualenv_with_conflict_fails(
             self, script):
         """

tests/functional/test_install_wheel.py

@@ -233,6 +233,7 @@ def test_wheel_record_lines_in_deterministic_order(script, data):
     assert record_lines == sorted(record_lines)
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_install_user_wheel(script, data, with_wheel):
     """
     Test user install from wheel (that has a script)

tests/functional/test_list.py

@@ -94,6 +94,7 @@ def test_local_columns_flag(simple_script):
 
 
 @pytest.mark.network
+@pytest.mark.incompatible_with_test_venv
 def test_user_flag(script, data):
     """
     Test the behavior of --user flag in the list command
@@ -110,6 +111,7 @@ def test_user_flag(script, data):
 
 
 @pytest.mark.network
+@pytest.mark.incompatible_with_test_venv
 def test_user_columns_flag(script, data):
     """
     Test the behavior of --user --format=columns flags in the list command
@@ -502,6 +504,7 @@ def test_list_path(tmpdir, script, data):
     assert {'name': 'simple', 'version': '2.0'} in json_result
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_list_path_exclude_user(tmpdir, script, data):
     """
     Test list with --path and make sure packages from --user are not picked

tests/functional/test_uninstall_user.py

@@ -9,6 +9,7 @@
 from tests.lib import assert_all_changes, pyversion
 
 
+@pytest.mark.incompatible_with_test_venv
 class Tests_UninstallUserSite:
 
     @pytest.mark.network

tests/unit/test_build_env.py

@@ -162,6 +162,7 @@ def test_build_env_overlay_prefix_has_priority(script):
     assert result.stdout.strip() == '2.0', str(result)
 
 
+@pytest.mark.incompatible_with_test_venv
 def test_build_env_isolation(script):
 
     # Create dummy `pkg` wheel.