Skip to content

Commit

Permalink
parent 8ae44bc
Browse files Browse the repository at this point in the history
author Dan Ryan <[email protected]> 1554074378 -0400
committer Dan Ryan <[email protected]> 1558982736 -0400
gpgsig -----BEGIN PGP SIGNATURE-----

 iQIzBAABCgAdFiEEb6jpcpb+5zzDideCsyDQTvvOpJUFAlzsMFAACgkQsyDQTvvO
 pJWZmRAAtge5wdprlLnKdWUYK5USZb7Uk3zJHi9UIvnO7nKP6UA/L2D/5nxZitvx
 pZI7cGG+8sLp2yZNtQZdW6LNM1jmRXgzdMLYQh/5zo5gbj6KLOw7erh0FU3L3uM6
 wNBNM135Eqt7b+4a4C5TEK2UjwRxBHAsF+3ZzUy+UJQqgQEKxFVxW4gC4yxpfMtL
 jipE8ludwuOIM88ZJapmLpv2R6adQTxWZedTlmczdsy2/WKGHTCCpWs96PBbntdI
 pVBmoXfMhgZi+IuGR3iBYU0qS97vjJ8Te9tQZAaB9JGSqv3hHDWo1ht/rrG2RXzp
 3i0Cf2vG4035EUh56VYE9FCC9m6Vu3U9iIR34BZG9K5+lDP7pmJmjT+GymEgMP0N
 GoP3LYUO+dJjMjaUEMsC6QIi6DAots3uk4lxIw3wcA4Im/N/i5xafsRj1Eu3UdBL
 wBDKMz/FQjH+tD+mnvTlzaxD5vdhhCdBu1gK59rjNMlzg8hz6EF61QbHCaQHd4UI
 VOGIa8ThLlLI3addxzq/McceAc+OsLJ9hm06jkjvvoIuKrHyE3DybdbYQC7uEwyw
 2AvVuMDCPcciYQnkJhNTKmGvPcUDYD7cF91GKcUJKdPdyzDCeFEo5SUTKnLV4Cj2
 VD2sdVUk4jnmYfE4pCFvHKYooxQDMKQk/VCKl1c9QHL9/ijCx+o=
 =b+H1
 -----END PGP SIGNATURE-----

Ensure resolver doesn't compare editable specifiers

- Don't compare versions of editable dependencies when updating using
  `--keep-outdated` -- editable dependencies will now be updated to
  the latest version
- Ensure we don't drop markers from the lockfile when versions are not
  updated
- Fixes #3656
- Fixes #3659

Signed-off-by: Dan Ryan <[email protected]>

Add future import for print function

Signed-off-by: Dan Ryan <[email protected]>

Handle all possible markers in lockfiles

Signed-off-by: Dan Ryan <[email protected]>

Fix json import

Signed-off-by: Dan Ryan <[email protected]>

point to correct reference for lockfile

Signed-off-by: Dan Ryan <[email protected]>

Fix marker merging errors

Signed-off-by: Dan Ryan <[email protected]>

Prevent automatically setting `editable=True`

- Fixes #3647

Signed-off-by: Dan Ryan <[email protected]>

Add new feature toggle for VCS dependency resolution

- Fixes #3577

Signed-off-by: Dan Ryan <[email protected]>

Fix syntax error

Signed-off-by: Dan Ryan <[email protected]>

Use string for environment

Signed-off-by: Dan Ryan <[email protected]>

Fix class name resolution for py27

Signed-off-by: Dan Ryan <[email protected]>

Write json files as unicode

Signed-off-by: Dan Ryan <[email protected]>

Fix resolution with env var

Signed-off-by: Dan Ryan <[email protected]>
  • Loading branch information
techalchemy committed May 27, 2019
1 parent 8ae44bc commit 4b09283
Show file tree
Hide file tree
Showing 10 changed files with 84 additions and 17 deletions.
1 change: 1 addition & 0 deletions news/3577.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies.
1 change: 1 addition & 0 deletions news/3647.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies.
2 changes: 2 additions & 0 deletions news/3656.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies.
- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated.
9 changes: 5 additions & 4 deletions pipenv/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -1394,6 +1394,8 @@ def pip_install(
src_dir = os.environ["PIP_SRC"]
src = ["--src", os.environ["PIP_SRC"]]
if not requirement.editable:
# Leave this off becauase old lockfiles don't have all deps included
# TODO: When can it be turned back on?
no_deps = False

if src_dir is not None:
Expand All @@ -1412,7 +1414,7 @@ def pip_install(
prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir,
delete=False
)
line = "-e" if requirement.editable else ""
line = "-e " if requirement.editable else ""
if requirement.editable or requirement.name is not None:
name = requirement.name
if requirement.extras:
Expand Down Expand Up @@ -1640,7 +1642,6 @@ def system_which(command, mult=False):
return result



def format_help(help):
"""Formats the help string."""
help = help.replace("Options:", str(crayons.normal("Options:", bold=True)))
Expand Down Expand Up @@ -1784,8 +1785,8 @@ def do_py(system=False):
),
err=True,
)
return
return

try:
click.echo(which("python", allow_global=system))
except AttributeError:
Expand Down
9 changes: 9 additions & 0 deletions pipenv/environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,15 @@ def _is_env_truthy(name):
NOTE: This only affects the ``install`` and ``uninstall`` commands.
"""

PIPENV_RESOLVE_VCS = _is_env_truthy(os.environ.get("PIPENV_RESOLVE_VCS", 'true'))
"""Tells Pipenv whether to resolve all VCS dependencies in full.
As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full.
To retain this behavior and avoid handling any conflicts that arise from the new
approach, you may set this to '0', 'off', or 'false'.
"""


PIPENV_PYUP_API_KEY = os.environ.get(
"PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0"
)
Expand Down
2 changes: 1 addition & 1 deletion pipenv/patched/piptools/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def clean_requires_python(candidates):
all_candidates = []
py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
for c in candidates:
if c.requires_python:
if getattr(c, "requires_python", None):
# Old specifications had people setting this to single digits
# which is effectively the same as '>=digit,<digit+1'
if c.requires_python.isdigit():
Expand Down
27 changes: 24 additions & 3 deletions pipenv/resolver.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# -*- coding: utf-8 -*-

from __future__ import absolute_import, print_function
import json
import logging
import os
Expand Down Expand Up @@ -106,9 +109,14 @@ def __init__(self, name, entry_dict, project, resolver, reverse_deps=None, dev=F
self.pipfile = project.parsed_pipfile.get(pipfile_section, {})
self.lockfile = project.lockfile_content.get(section, {})
self.pipfile_dict = self.pipfile.get(self.pipfile_name, {})
self.lockfile_dict = self.lockfile.get(name, entry_dict)
if self.dev and self.name in project.lockfile_content.get("default", {}):
self.lockfile_dict = project.lockfile_content["default"][name]
else:
self.lockfile_dict = self.lockfile.get(name, entry_dict)
self.resolver = resolver
self.reverse_deps = reverse_deps
self._original_markers = None
self._markers = None
self._entry = None
self._lockfile_entry = None
self._pipfile_entry = None
Expand Down Expand Up @@ -232,6 +240,10 @@ def get_cleaned_dict(self, keep_outdated=False):
entry_extras.extend(list(self.lockfile_entry.extras))
self._entry.req.extras = entry_extras
self.entry_dict["extras"] = self.entry.extras
if self.original_markers and not self.markers:
original_markers = self.marker_to_str(self.original_markers)
self.markers = original_markers
self.entry_dict["markers"] = self.marker_to_str(original_markers)
entry_hashes = set(self.entry.hashes)
locked_hashes = set(self.lockfile_entry.hashes)
if entry_hashes != locked_hashes and not self.is_updated:
Expand All @@ -248,6 +260,10 @@ def lockfile_entry(self):
self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict)
return self._lockfile_entry

@lockfile_entry.setter
def lockfile_entry(self, entry):
self._lockfile_entry = entry

@property
def pipfile_entry(self):
if self._pipfile_entry is None:
Expand Down Expand Up @@ -359,6 +375,7 @@ def updated_version(self):

@property
def updated_specifier(self):
# type: () -> str
return self.entry.specifiers

@property
Expand All @@ -373,7 +390,7 @@ def original_version(self):
return None

def validate_specifiers(self):
if self.is_in_pipfile:
if self.is_in_pipfile and not self.pipfile_entry.editable:
return self.pipfile_entry.requirement.specifier.contains(self.updated_version)
return True

Expand Down Expand Up @@ -550,8 +567,11 @@ def validate_constraints(self):
constraint.check_if_exists(False)
except Exception:
from pipenv.exceptions import DependencyConflict
from pipenv.environments import is_verbose
if is_verbose():
print("Tried constraint: {0!r}".format(constraint), file=sys.stderr)
msg = (
"Cannot resolve conflicting version {0}{1} while {1}{2} is "
"Cannot resolve conflicting version {0}{1} while {2}{3} is "
"locked.".format(
self.name, self.updated_specifier, self.old_name, self.old_specifiers
)
Expand Down Expand Up @@ -624,6 +644,7 @@ def clean_results(results, resolver, project, dev=False):

def clean_outdated(results, resolver, project, dev=False):
from pipenv.vendor.requirementslib.models.requirements import Requirement
from pipenv.environments import is_verbose
if not project.lockfile_exists:
return results
lockfile = project.lockfile_content
Expand Down
29 changes: 21 additions & 8 deletions pipenv/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@
import parse

from . import environments
from .exceptions import PipenvUsageError, ResolutionFailure, RequirementError, PipenvCmdError
from .exceptions import (
PipenvUsageError, RequirementError, PipenvCmdError, ResolutionFailure
)
from .pep508checker import lookup
from .vendor.urllib3 import util as urllib3_util

Expand Down Expand Up @@ -398,7 +400,6 @@ def parse_line(
):
# type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]]
from .vendor.requirementslib.models.requirements import Requirement
from .exceptions import ResolutionFailure
if index_lookup is None:
index_lookup = {}
if markers_lookup is None:
Expand Down Expand Up @@ -444,6 +445,7 @@ def get_deps_from_req(cls, req, resolver=None):
# type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
from .vendor.requirementslib.models.requirements import Requirement
from requirementslib.utils import is_installable_dir
constraints = set() # type: Set[str]
locked_deps = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]]
if (req.is_file_or_url or req.is_vcs) and not req.is_wheel:
Expand All @@ -463,7 +465,16 @@ def get_deps_from_req(cls, req, resolver=None):
setup_info = req.req.setup_info
setup_info.get_info()
locked_deps[pep423_name(name)] = entry
requirements = [v for v in getattr(setup_info, "requires", {}).values()]
requirements = []
# Allow users to toggle resolution off for non-editable VCS packages
# but leave it on for local, installable folders on the filesystem
if environments.PIPENV_RESOLVE_VCS or (
req.editable or parsed_line.is_wheel or (
req.is_file_or_url and parsed_line.is_local and
is_installable_dir(parsed_line.path)
)
):
requirements = [v for v in getattr(setup_info, "requires", {}).values()]
for r in requirements:
if getattr(r, "url", None) and not getattr(r, "editable", False):
if r is not None:
Expand Down Expand Up @@ -1797,13 +1808,15 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):

# If a package is **PRESENT** in the pipfile but has no markers, make sure we
# **NEVER** include markers in the lockfile
if "markers" in dep:
if "markers" in dep and dep.get("markers", "").strip():
# First, handle the case where there is no top level dependency in the pipfile
if not is_top_level:
try:
lockfile["markers"] = translate_markers(dep)["markers"]
except TypeError:
pass
translated = translate_markers(dep).get("markers", "").strip()
if translated:
try:
lockfile["markers"] = translated
except TypeError:
pass
# otherwise make sure we are prioritizing whatever the pipfile says about the markers
# If the pipfile says nothing, then we should put nothing in the lockfile
else:
Expand Down
2 changes: 1 addition & 1 deletion tasks/vendoring/patches/patched/piptools.patch
Original file line number Diff line number Diff line change
Expand Up @@ -608,7 +608,7 @@ index 9b4b4c2..8875543 100644
+ all_candidates = []
+ py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
+ for c in candidates:
+ if c.requires_python:
+ if getattr(c, "requires_python", None):
+ # Old specifications had people setting this to single digits
+ # which is effectively the same as '>=digit,<digit+1'
+ if c.requires_python.isdigit():
Expand Down
19 changes: 19 additions & 0 deletions tests/integration/test_lock.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
# -*- coding: utf-8 -*-

import json
import os
import sys

import pytest

from flaky import flaky
from vistir.compat import Path
from vistir.misc import to_text
from pipenv.utils import temp_environ


Expand Down Expand Up @@ -122,6 +126,21 @@ def test_keep_outdated_doesnt_upgrade_pipfile_pins(PipenvInstance, pypi):
assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1"


def test_keep_outdated_keeps_markers_not_removed(PipenvInstance, pypi):
with PipenvInstance(chdir=True, pypi=pypi) as p:
c = p.pipenv("install tablib")
assert c.ok
lockfile = Path(p.lockfile_path)
lockfile_content = lockfile.read_text()
lockfile_json = json.loads(lockfile_content)
assert "tablib" in lockfile_json["default"]
lockfile_json["default"]["tablib"]["markers"] = "python_version >= '2.7'"
lockfile.write_text(to_text(json.dumps(lockfile_json)))
c = p.pipenv("lock --keep-outdated")
assert c.ok
assert p.lockfile["default"]["tablib"].get("markers", "") == "python_version >= '2.7'"


@pytest.mark.lock
@pytest.mark.keep_outdated
def test_keep_outdated_doesnt_update_satisfied_constraints(PipenvInstance, pypi):
Expand Down

0 comments on commit 4b09283

Please sign in to comment.