From 483621f4789699a4a431a1cf2bdaa6c8cb256157 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 00:10:46 +0800 Subject: [PATCH 01/12] recognize editable packages as in project Signed-off-by: Frost Ming --- pipenv/environment.py | 5 ++--- tests/integration/test_project.py | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/pipenv/environment.py b/pipenv/environment.py index 7a7fd2ea02..8ac78dd8b8 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -246,7 +246,7 @@ def get_distributions(self): def find_egg(self, egg_dist): """Find an egg by name in the given environment""" - site_packages = get_python_lib() + site_packages = self.libdir[1] search_filename = "{0}.egg-link".format(egg_dist.project_name) try: user_site = site.getusersitepackages() @@ -264,8 +264,7 @@ def locate_dist(self, dist): If the egg - link doesn 't exist, return the supplied distribution.""" location = self.find_egg(dist) - if not location: - return dist.location + return location or dist.location def dist_is_in_project(self, dist): """Determine whether the supplied distribution is in the environment.""" diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 5e1bafb919..aa72673d76 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -2,6 +2,7 @@ import io import pytest import os +import tarfile from pipenv.project import Project from pipenv.utils import temp_environ from pipenv.patched import pipfile @@ -161,3 +162,21 @@ def test_rewrite_outline_table(PipenvInstance, pypi): contents = f.read() assert "[packages.requests]" not in contents assert 'requests = {version = "*"}' in contents + + +@pytest.mark.install +@pytest.mark.project +def test_include_editable_packages(PipenvInstance, pypi, testsroot, tmpdir): + file_name = "requests-2.19.1.tar.gz" + package = os.path.join(tmpdir, "requests-2.19.1") + source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) + with PipenvInstance(chdir=True, pypi=pypi) as p: + with tarfile.open(source_path, "r:gz") as tarinfo: + tarinfo.extractall(path=tmpdir) + c = p.pipenv('install -e {}'.format(package)) + assert c.return_code == 0 + project = Project() + assert "requests" in [ + package.project_name + for package in project.environment.get_installed_packages() + ] From ff8d17f383882305aea584e55e5d6b209a65e063 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 08:59:06 +0800 Subject: [PATCH 02/12] switch to pathlib tmpdir Signed-off-by: Frost Ming --- tests/integration/test_project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index aa72673d76..fce8ea6145 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -166,13 +166,13 @@ def test_rewrite_outline_table(PipenvInstance, pypi): @pytest.mark.install @pytest.mark.project -def test_include_editable_packages(PipenvInstance, pypi, testsroot, tmpdir): +def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpdir): file_name = "requests-2.19.1.tar.gz" - package = os.path.join(tmpdir, "requests-2.19.1") + package = pathlib_tmpdir.joinpath("requests-2.19.1") source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) with PipenvInstance(chdir=True, pypi=pypi) as p: with tarfile.open(source_path, "r:gz") as tarinfo: - tarinfo.extractall(path=tmpdir) + tarinfo.extractall(path=str(pathlib_tmpdir)) c = p.pipenv('install -e {}'.format(package)) assert c.return_code == 0 project = Project() From 5d8ac0901c1feff1d9a6e425d38a2d17bb6810d9 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 09:37:32 +0800 Subject: [PATCH 03/12] add news Signed-off-by: Frost Ming --- news/3240.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/3240.bugfix.rst diff --git a/news/3240.bugfix.rst b/news/3240.bugfix.rst new file mode 100644 index 0000000000..88375bf9c0 --- /dev/null +++ b/news/3240.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug that editable pacakges can't be uninstalled correctly. From 95c70c3f618febf02e8bb83ffd84c8e531101ae5 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 11:37:22 +0800 Subject: [PATCH 04/12] Correct the virtualenv location passed to environment Signed-off-by: Frost Ming --- pipenv/core.py | 5 +---- pipenv/environment.py | 2 +- pipenv/environments.py | 5 +++++ pipenv/project.py | 5 +++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index f815e5de9b..28d5f9a84c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1566,10 +1566,7 @@ def gen(out): def warn_in_virtualenv(): # Only warn if pipenv isn't already active. - pipenv_active = os.environ.get("PIPENV_ACTIVE") - if (environments.PIPENV_USE_SYSTEM or environments.PIPENV_VIRTUALENV) and not ( - pipenv_active or environments.is_quiet() - ): + if environments.is_in_virtualenv() and not environments.is_quiet(): click.echo( "{0}: Pipenv found itself running within a virtual environment, " "so it will automatically use that environment, instead of " diff --git a/pipenv/environment.py b/pipenv/environment.py index 8ac78dd8b8..5f32747cd2 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -30,7 +30,7 @@ def __init__(self, prefix=None, is_venv=False, base_working_set=None, pipfile=No self._modules = {'pkg_resources': pkg_resources, 'pipenv': pipenv} self.base_working_set = base_working_set if base_working_set else BASE_WORKING_SET prefix = normalize_path(prefix) - self.is_venv = not prefix == normalize_path(sys.prefix) + self.is_venv = is_venv or prefix != normalize_path(sys.prefix) if not sources: sources = [] self.project = project diff --git a/pipenv/environments.py b/pipenv/environments.py index 066c296d58..2d9bfaa868 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -276,6 +276,11 @@ def is_quiet(threshold=-1): return PIPENV_VERBOSITY <= threshold +def is_in_virtualenv(): + pipenv_active = os.environ.get("PIPENV_ACTIVE") + return (PIPENV_USE_SYSTEM or PIPENV_VIRTUALENV) and not pipenv_active + + PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/project.py b/pipenv/project.py index 590fafe53f..fa7adb3ecf 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -47,6 +47,7 @@ PIPENV_TEST_INDEX, PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, + is_in_virtualenv ) @@ -345,8 +346,8 @@ def pipfile_package_names(self): @property def environment(self): if not self._environment: - prefix = self.get_location_for_virtualenv() - is_venv = prefix == sys.prefix + prefix = self.virtualenv_location + is_venv = is_in_virtualenv() sources = self.sources if self.sources else [DEFAULT_SOURCE,] self._environment = Environment( prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, From 748d22b2f23dd8e6ca5bb6d3462cb279a6d0b3a2 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 13:08:12 +0800 Subject: [PATCH 05/12] virtualenv fixture and a test case Signed-off-by: Frost Ming --- news/3231.bugfix.rst | 1 + pipenv/environments.py | 3 ++- pipenv/project.py | 6 +++--- tests/integration/conftest.py | 21 +++++++++++++++++++++ tests/integration/test_project.py | 16 ++++++++++++++++ 5 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 news/3231.bugfix.rst diff --git a/news/3231.bugfix.rst b/news/3231.bugfix.rst new file mode 100644 index 0000000000..036b2c1210 --- /dev/null +++ b/news/3231.bugfix.rst @@ -0,0 +1 @@ +Correctly detect the virtualenv location inside an activated virtualenv. diff --git a/pipenv/environments.py b/pipenv/environments.py index 2d9bfaa868..45a4f6b857 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -278,7 +278,8 @@ def is_quiet(threshold=-1): def is_in_virtualenv(): pipenv_active = os.environ.get("PIPENV_ACTIVE") - return (PIPENV_USE_SYSTEM or PIPENV_VIRTUALENV) and not pipenv_active + virtual_env = os.environ.get("VIRTUAL_ENV") + return (PIPENV_USE_SYSTEM or virtual_env) and not pipenv_active PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") diff --git a/pipenv/project.py b/pipenv/project.py index fa7adb3ecf..334782d965 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -43,7 +43,6 @@ PIPENV_MAX_DEPTH, PIPENV_PIPFILE, PIPENV_VENV_IN_PROJECT, - PIPENV_VIRTUALENV, PIPENV_TEST_INDEX, PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, @@ -427,8 +426,9 @@ def virtualenv_name(self): @property def virtualenv_location(self): # if VIRTUAL_ENV is set, use that. - if PIPENV_VIRTUALENV: - return PIPENV_VIRTUALENV + virtualenv_env = os.getenv("VIRTUAL_ENV") + if virtualenv_env: + return virtualenv_env if not self._virtualenv_location: # Use cached version, if available. assert self.project_directory, "project not created" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 6ed95b3e7c..0b27efe80f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,6 +6,8 @@ import pytest from pipenv._compat import TemporaryDirectory, Path +from pipenv.exceptions import VirtualenvActivationException +from pipenv.utils import temp_environ from pipenv.vendor import delegator from pipenv.vendor import requests from pipenv.vendor import toml @@ -239,3 +241,22 @@ def finalize(): @pytest.fixture() def testsroot(): return TESTS_ROOT + + +@pytest.fixture() +def virtualenv(pathlib_tmpdir): + virtualenv_path = pathlib_tmpdir / "venv" + with temp_environ(): + c = delegator.run("virtualenv {}".format(virtualenv_path), block=True) + assert c.return_code == 0 + for name in ("bin", "Scripts"): + activate_this = virtualenv_path / name / "activate_this.py" + if activate_this.exists(): + with open(str(activate_this)) as f: + code = compile(f.read(), str(activate_this), "exec") + exec(code, dict(__file__=str(activate_this))) + break + else: + raise VirtualenvActivationException("Can't find the activate_this.py script.") + os.environ["VIRTUAL_ENV"] = str(virtualenv_path) + yield virtualenv_path diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index fce8ea6145..f49518868f 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -180,3 +180,19 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd package.project_name for package in project.environment.get_installed_packages() ] + + +@pytest.mark.project +def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): + with PipenvInstance(chdir=True, pypi=pypi) as p: + project = Project() + assert project.virtualenv_location == str(virtualenv) + c = p.pipenv("run pip install click") + assert c.return_code == 0 + assert "Courtesy Notice" in c.err + c = p.pipenv('run python -c "import click;print(click.__file__)"') + assert c.return_code == 0 + assert c.out.strip().startswith(str(virtualenv)) + c = p.pipenv("clean --dry-run") + assert c.return_code == 0 + assert "click" in c.out From a8f52eb0a6a658d6224ccd6063cb8acd5ca34fb9 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 17:51:26 +0800 Subject: [PATCH 06/12] determin virtualenv location on the run Signed-off-by: Frost Ming --- pipenv/project.py | 4 +++- tests/integration/test_project.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/project.py b/pipenv/project.py index f3a11f6972..b2da4fd218 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -46,6 +46,7 @@ PIPENV_TEST_INDEX, PIPENV_PYTHON, PIPENV_DEFAULT_PYTHON_VERSION, + PIPENV_IGNORE_VIRTUALENVS, is_in_virtualenv ) @@ -427,7 +428,8 @@ def virtualenv_name(self): def virtualenv_location(self): # if VIRTUAL_ENV is set, use that. virtualenv_env = os.getenv("VIRTUAL_ENV") - if virtualenv_env: + if ("PIPENV_ACTIVE" not in os.environ and + not PIPENV_IGNORE_VIRTUALENVS and virtualenv_env): return virtualenv_env if not self._virtualenv_location: # Use cached version, if available. diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index f49518868f..8bb76da16f 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -185,6 +185,7 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd @pytest.mark.project def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): with PipenvInstance(chdir=True, pypi=pypi) as p: + os.environ.pop("PIPENV_IGNORE_VIRTUALENVS", None) project = Project() assert project.virtualenv_location == str(virtualenv) c = p.pipenv("run pip install click") From 915b3c13345b847a71aac137165c02743747c3a0 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 18:23:28 +0800 Subject: [PATCH 07/12] ignore PIPENV_ACTIVE in testing Signed-off-by: Frost Ming --- tests/integration/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 0b27efe80f..3e13928920 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -113,6 +113,8 @@ def isolate(pathlib_tmpdir): os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") mkdir_p(os.path.join(home_dir, ".virtualenvs")) os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + # Ignore PIPENV_ACTIVE so that it works as under a bare environment. + os.environ.pop("PIPENV_ACTIVE", None) global WE_HAVE_GITHUB_SSH_KEYS WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() From 32f5d85b0b467648597855ed54a8c55137397661 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 19:42:43 +0800 Subject: [PATCH 08/12] Pop VIRTUAL_ENV env Signed-off-by: Frost Ming --- tests/integration/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 3e13928920..a2f1c2e7ed 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -115,6 +115,7 @@ def isolate(pathlib_tmpdir): os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) # Ignore PIPENV_ACTIVE so that it works as under a bare environment. os.environ.pop("PIPENV_ACTIVE", None) + os.environ.pop("VIRTUAL_ENV", None) global WE_HAVE_GITHUB_SSH_KEYS WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() From 6901eae31d5433da5edd3c77aa8825712ec7faea Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sat, 17 Nov 2018 22:50:04 +0800 Subject: [PATCH 09/12] expand env before run script Signed-off-by: Frost Ming --- news/3178.bugfix.rst | 1 + pipenv/core.py | 6 ++++-- tests/integration/test_run.py | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 news/3178.bugfix.rst diff --git a/news/3178.bugfix.rst b/news/3178.bugfix.rst new file mode 100644 index 0000000000..d3a4fd6ccb --- /dev/null +++ b/news/3178.bugfix.rst @@ -0,0 +1 @@ +Environment variables are expanded correctly before running scripts on POSIX. diff --git a/pipenv/core.py b/pipenv/core.py index 7624db8b8f..94212875e5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2262,7 +2262,7 @@ def do_run_nt(script): def do_run_posix(script, command): - command_path = system_which(script.command) + command_path = system_which(os.path.expandvars(script.command)) if not command_path: if project.has_script(command): click.echo( @@ -2287,7 +2287,9 @@ def do_run_posix(script, command): err=True, ) sys.exit(1) - os.execl(command_path, command_path, *script.args) + os.execl( + command_path, command_path, *[os.path.expandvars(arg) for arg in script.args] + ) def do_run(command, args, three=None, python=False, pypi_mirror=None): diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 72f10596df..8167a31811 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -1,6 +1,7 @@ import os from pipenv.project import Project +from pipenv.utils import temp_environ import pytest @@ -27,6 +28,7 @@ def test_scripts(PipenvInstance): notfoundscript = "randomthingtotally" appendscript = "cmd arg1" multicommand = "bash -c \"cd docs && make html\"" +scriptwithenv = "echo $HELLO" """) c = p.pipenv('install') assert c.return_code == 0 @@ -52,3 +54,9 @@ def test_scripts(PipenvInstance): script = project.build_script('appendscript', ['a', 'b']) assert script.command == 'cmd' assert script.args == ['arg1', 'a', 'b'] + + with temp_environ(): + os.environ['HELLO'] = 'WORLD' + c = p.pipenv("run scriptwithenv") + assert c.ok + assert c.out.strip() == "WORLD" From 5e94bcb595035d865b301855c75fc5ca83d2ab51 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sun, 18 Nov 2018 08:56:43 +0800 Subject: [PATCH 10/12] fix case for windows Signed-off-by: Frost Ming --- tests/integration/test_run.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 8167a31811..205ccadd36 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -28,8 +28,11 @@ def test_scripts(PipenvInstance): notfoundscript = "randomthingtotally" appendscript = "cmd arg1" multicommand = "bash -c \"cd docs && make html\"" -scriptwithenv = "echo $HELLO" """) + if os.name == "nt": + f.write('scriptwithenv = "echo %HELLO%"\n') + else: + f.write('scriptwithenv = "echo $HELLO"\n') c = p.pipenv('install') assert c.return_code == 0 From e909d9b84f8578284e4a6ebe5154de8ad4b5d5a9 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sun, 18 Nov 2018 09:42:04 +0800 Subject: [PATCH 11/12] do not check on windows Signed-off-by: Frost Ming --- tests/integration/test_run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 205ccadd36..8e82a6e2cb 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -62,4 +62,5 @@ def test_scripts(PipenvInstance): os.environ['HELLO'] = 'WORLD' c = p.pipenv("run scriptwithenv") assert c.ok - assert c.out.strip() == "WORLD" + if os.name != "nt": # This doesn't work on CI windows. + assert c.out.strip() == "WORLD" From 52499dfa9aec22886fe0172f2bfe375a00bbc0b0 Mon Sep 17 00:00:00 2001 From: frostming Date: Mon, 19 Nov 2018 10:47:19 +0800 Subject: [PATCH 12/12] don't expand var for command --- pipenv/core.py | 2 +- pipenv/environment.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 94212875e5..0cbd23e3a5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -2262,7 +2262,7 @@ def do_run_nt(script): def do_run_posix(script, command): - command_path = system_which(os.path.expandvars(script.command)) + command_path = system_which(script.command) if not command_path: if project.has_script(command): click.echo( diff --git a/pipenv/environment.py b/pipenv/environment.py index 5f32747cd2..024faf46be 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -482,7 +482,6 @@ def activated(self, include_extras=True, extra_dists=None): ]) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - os.environ["PATH"] = self.base_paths["PATH"] os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] if self.is_venv: os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix)