Merge pull request #3107 from abn/issue/3079

Improved support for system environment

poetry/installation/pip_installer.py

@@ -113,7 +113,9 @@ def remove(self, package):
             raise
 
         # This is a workaround for https://github.com/pypa/pip/issues/4176
-        nspkg_pth_file = self._env.site_packages / "{}-nspkg.pth".format(package.name)
+        nspkg_pth_file = self._env.site_packages.path / "{}-nspkg.pth".format(
+            package.name
+        )
         if nspkg_pth_file.exists():
             nspkg_pth_file.unlink()
 

poetry/masonry/builders/editable.py

@@ -13,6 +13,7 @@
 from poetry.utils._compat import WINDOWS
 from poetry.utils._compat import Path
 from poetry.utils._compat import decode
+from poetry.utils.helpers import is_dir_writable
 
 
 SCRIPT_TEMPLATE = """\
@@ -94,7 +95,6 @@ def _setup_build(self):
                 os.remove(str(setup))
 
     def _add_pth(self):
-        pth_file = Path(self._module.name).with_suffix(".pth")
         paths = set()
         for include in self._module.includes:
             if isinstance(include, PackageInclude) and (
@@ -106,34 +106,40 @@ def _add_pth(self):
         for path in paths:
             content += decode(path + os.linesep)
 
-        for site_package in [self._env.site_packages, self._env.usersite]:
-            if not site_package:
-                continue
-
-            try:
-                site_package.mkdir(parents=True, exist_ok=True)
-                path = site_package.joinpath(pth_file)
-                self._debug(
-                    "  - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
-                        path.name, site_package, self._poetry.file.parent
-                    )
+        pth_file = Path(self._module.name).with_suffix(".pth")
+        try:
+            pth_file = self._env.site_packages.write_text(
+                pth_file, content, encoding="utf-8"
+            )
+            self._debug(
+                "  - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
+                    pth_file.name, pth_file.parent, self._poetry.file.parent
                 )
-                path.write_text(content, encoding="utf-8")
-                return [path]
-            except PermissionError:
-                self._debug("- <b>{}</b> is not writable trying next available site")
-
-        self._io.error_line(
-            "  - Failed to create <c2>{}</c2> for {}".format(
-                pth_file.name, self._poetry.file.parent
             )
-        )
-        return []
+            return [pth_file]
+        except OSError:
+            # TODO: Replace with PermissionError
+            self._io.error_line(
+                "  - Failed to create <c2>{}</c2> for {}".format(
+                    pth_file.name, self._poetry.file.parent
+                )
+            )
+            return []
 
     def _add_scripts(self):
         added = []
         entry_points = self.convert_entry_points()
-        scripts_path = Path(self._env.paths["scripts"])
+
+        for scripts_path in self._env.script_dirs:
+            if is_dir_writable(path=scripts_path, create=True):
+                break
+        else:
+            self._io.error_line(
+                "  - Failed to find a suitable script installation directory for {}".format(
+                    self._poetry.file.parent
+                )
+            )
+            return []
 
         scripts = entry_points.get("console_scripts", [])
         for script in scripts:
@@ -151,7 +157,7 @@ def _add_scripts(self):
                 f.write(
                     decode(
                         SCRIPT_TEMPLATE.format(
-                            python=self._env._bin("python"),
+                            python=self._env.python,
                             module=module,
                             callable_holder=callable_holder,
                             callable_=callable_,
@@ -165,9 +171,7 @@ def _add_scripts(self):
 
             if WINDOWS:
                 cmd_script = script_file.with_suffix(".cmd")
-                cmd = WINDOWS_CMD_TEMPLATE.format(
-                    python=self._env._bin("python"), script=name
-                )
+                cmd = WINDOWS_CMD_TEMPLATE.format(python=self._env.python, script=name)
                 self._debug(
                     "  - Adding the <c2>{}</c2> script wrapper to <b>{}</b>".format(
                         cmd_script.name, scripts_path
@@ -187,19 +191,27 @@ def _add_dist_info(self, added_files):
         added_files = added_files[:]
 
         builder = WheelBuilder(self._poetry)
-        dist_info = self._env.site_packages.joinpath(builder.dist_info)
+
+        dist_info_path = Path(builder.dist_info)
+        for dist_info in self._env.site_packages.find(
+            dist_info_path, writable_only=True
+        ):
+            if dist_info.exists():
+                self._debug(
+                    "  - Removing existing <c2>{}</c2> directory from <b>{}</b>".format(
+                        dist_info.name, dist_info.parent
+                    )
+                )
+                shutil.rmtree(str(dist_info))
+
+        dist_info = self._env.site_packages.mkdir(dist_info_path)
 
         self._debug(
             "  - Adding the <c2>{}</c2> directory to <b>{}</b>".format(
-                dist_info.name, self._env.site_packages
+                dist_info.name, dist_info.parent
             )
         )
 
-        if dist_info.exists():
-            shutil.rmtree(str(dist_info))
-
-        dist_info.mkdir()
-
         with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f:
             builder._write_metadata_file(f)
 

poetry/utils/env.py

@@ -39,6 +39,8 @@
 from poetry.utils._compat import encode
 from poetry.utils._compat import list_to_shell_command
 from poetry.utils._compat import subprocess
+from poetry.utils.helpers import is_dir_writable
+from poetry.utils.helpers import paths_csv
 
 
 GET_ENVIRONMENT_INFO = """\
@@ -143,6 +145,120 @@ def _version_nodot(version):
 """
 
 
+class SitePackages:
+    def __init__(
+        self, path, fallbacks=None, skip_write_checks=False
+    ):  # type: (Path, List[Path], bool) -> None
+        self._path = path
+        self._fallbacks = fallbacks or []
+        self._skip_write_checks = skip_write_checks
+        self._candidates = [self._path] + self._fallbacks
+        self._writable_candidates = None if not skip_write_checks else self._candidates
+
+    @property
+    def path(self):  # type: () -> Path
+        return self._path
+
+    @property
+    def candidates(self):  # type: () -> List[Path]
+        return self._candidates
+
+    @property
+    def writable_candidates(self):  # type: () -> List[Path]
+        if self._writable_candidates is not None:
+            return self._writable_candidates
+
+        self._writable_candidates = []
+        for candidate in self._candidates:
+            if not is_dir_writable(path=candidate, create=True):
+                continue
+            self._writable_candidates.append(candidate)
+
+        return self._writable_candidates
+
+    def make_candidates(
+        self, path, writable_only=False
+    ):  # type: (Path, bool) -> List[Path]
+        candidates = self._candidates if not writable_only else self.writable_candidates
+        if path.is_absolute():
+            for candidate in candidates:
+                try:
+                    path.relative_to(candidate)
+                    return [path]
+                except ValueError:
+                    pass
+            else:
+                raise ValueError(
+                    "{} is not relative to any discovered {}sites".format(
+                        path, "writable " if writable_only else ""
+                    )
+                )
+
+        return [candidate / path for candidate in candidates if candidate]
+
+    def _path_method_wrapper(
+        self, path, method, *args, **kwargs
+    ):  # type: (Path, str, *Any, **Any) -> Union[Tuple[Path, Any], List[Tuple[Path, Any]]]
+
+        # TODO: Move to parameters after dropping Python 2.7
+        return_first = kwargs.pop("return_first", True)
+        writable_only = kwargs.pop("writable_only", False)
+
+        candidates = self.make_candidates(path, writable_only=writable_only)
+
+        if not candidates:
+            raise RuntimeError(
+                'Unable to find a suitable destination for "{}" in {}'.format(
+                    str(path), paths_csv(self._candidates)
+                )
+            )
+
+        results = []
+
+        for candidate in candidates:
+            try:
+                result = candidate, getattr(candidate, method)(*args, **kwargs)
+                if return_first:
+                    return result
+                else:
+                    results.append(result)
+            except (IOError, OSError):
+                # TODO: Replace with PermissionError
+                pass
+
+        if results:
+            return results
+
+        raise OSError("Unable to access any of {}".format(paths_csv(candidates)))
+
+    def write_text(self, path, *args, **kwargs):  # type: (Path, *Any, **Any) -> Path
+        return self._path_method_wrapper(path, "write_text", *args, **kwargs)[0]
+
+    def mkdir(self, path, *args, **kwargs):  # type: (Path, *Any, **Any) -> Path
+        return self._path_method_wrapper(path, "mkdir", *args, **kwargs)[0]
+
+    def exists(self, path):  # type: (Path) -> bool
+        return any(
+            value[-1]
+            for value in self._path_method_wrapper(path, "exists", return_first=False)
+        )
+
+    def find(self, path, writable_only=False):  # type: (Path, bool) -> List[Path]
+        return [
+            value[0]
+            for value in self._path_method_wrapper(
+                path, "exists", return_first=False, writable_only=writable_only
+            )
+            if value[-1] is True
+        ]
+
+    def __getattr__(self, item):
+        try:
+            return super(SitePackages, self).__getattribute__(item)
+        except AttributeError:
+            return getattr(self.path, item)
+
+
 class EnvError(Exception):
 
     pass
@@ -756,6 +872,7 @@ def __init__(self, path, base=None):  # type: (Path, Optional[Path]) -> None
         self._supported_tags = None
         self._purelib = None
         self._platlib = None
+        self._script_dirs = None
 
     @property
     def path(self):  # type: () -> Path
@@ -810,16 +927,25 @@ def pip_version(self):
         return self._pip_version
 
     @property
-    def site_packages(self):  # type: () -> Path
+    def site_packages(self):  # type: () -> SitePackages
         if self._site_packages is None:
-            self._site_packages = self.purelib
+            # we disable write checks if no user site exist
+            fallbacks = [self.usersite] if self.usersite else []
+            self._site_packages = SitePackages(
+                self.purelib, fallbacks, skip_write_checks=False if fallbacks else True
+            )
         return self._site_packages
 
     @property
     def usersite(self):  # type: () -> Optional[Path]
         if "usersite" in self.paths:
             return Path(self.paths["usersite"])
 
+    @property
+    def userbase(self):  # type: () -> Optional[Path]
+        if "userbase" in self.paths:
+            return Path(self.paths["userbase"])
+
     @property
     def purelib(self):  # type: () -> Path
         if self._purelib is None:
@@ -966,6 +1092,18 @@ def execute(self, bin, *args, **kwargs):
     def is_venv(self):  # type: () -> bool
         raise NotImplementedError()
 
+    @property
+    def script_dirs(self):  # type: () -> List[Path]
+        if self._script_dirs is None:
+            self._script_dirs = (
+                [Path(self.paths["scripts"])]
+                if "scripts" in self.paths
+                else self._bin_dir
+            )
+            if self.userbase:
+                self._script_dirs.append(self.userbase / self._script_dirs[0].name)
+        return self._script_dirs
+
     def _bin(self, bin):  # type: (str) -> str
         """
         Return path to the given executable.
@@ -1001,6 +1139,10 @@ class SystemEnv(Env):
     A system (i.e. not a virtualenv) Python environment.
     """
 
+    @property
+    def python(self):  # type: () -> str
+        return sys.executable
+
     @property
     def sys_path(self):  # type: () -> List[str]
         return sys.path
@@ -1041,6 +1183,7 @@ def get_paths(self):  # type: () -> Dict[str, str]
 
         if site.check_enableusersite() and hasattr(obj, "install_usersite"):
             paths["usersite"] = getattr(obj, "install_usersite")
+            paths["userbase"] = getattr(obj, "install_userbase")
 
         return paths
 
@@ -1176,7 +1319,7 @@ def is_venv(self):  # type: () -> bool
 
     def is_sane(self):
         # A virtualenv is considered sane if both "python" and "pip" exist.
-        return os.path.exists(self._bin("python")) and os.path.exists(self._bin("pip"))
+        return os.path.exists(self.python) and os.path.exists(self._bin("pip"))
 
     def _run(self, cmd, **kwargs):
         with self.temp_environ():

poetry/utils/helpers.py

@@ -5,6 +5,7 @@
 import tempfile
 
 from contextlib import contextmanager
+from typing import List
 from typing import Optional
 
 import requests
@@ -113,3 +114,22 @@ def get_package_version_display_string(
         )
 
     return package.full_pretty_version
+
+
+def paths_csv(paths):  # type: (List[Path]) -> str
+    return ", ".join('"{}"'.format(str(c)) for c in paths)
+
+
+def is_dir_writable(path, create=False):  # type: (Path, bool) -> bool
+    try:
+        if not path.exists():
+            if not create:
+                return False
+            path.mkdir(parents=True, exist_ok=True)
+
+        with tempfile.TemporaryFile(dir=str(path)):
+            pass
+    except (IOError, OSError):
+        return False
+    else:
+        return True

tests/installation/test_pip_installer.py

@@ -189,7 +189,9 @@ def test_uninstall_git_package_nspkg_pth_cleanup(mocker, tmp_venv, pool):
     )
 
     # we do this here because the virtual env might not be usable if failure case is triggered
-    pth_file_candidate = tmp_venv.site_packages / "{}-nspkg.pth".format(package.name)
+    pth_file_candidate = tmp_venv.site_packages.path / "{}-nspkg.pth".format(
+        package.name
+    )
 
     # in order to reproduce the scenario where the git source is removed prior to proper
     # clean up of nspkg.pth file, we need to make sure the fixture is copied and not