From 81b0d515a716ea264343febf50a60eda001bac7d Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 14 Apr 2021 12:02:54 -0400 Subject: [PATCH 1/8] Fixing issue with losing data about local files When a dependency is a local file we need to keep track of that information. During the combine_install_requirements function we were losing that information. This merge request looks for that information and combines it into the InstallRequirements object that is going to be returned by that function. --- piptools/resolver.py | 9 +++++++++ tests/test_data/fake-editables.json | 3 ++- .../fake_package_a-0.1-py2.py3-none-any.whl | Bin 0 -> 1631 bytes .../packages/fake_with_local_files/setup.py | 12 ++++++++++++ tests/test_resolver.py | 5 +++++ 5 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 tests/test_data/non_linked_wheel_file/fake_package_a-0.1-py2.py3-none-any.whl create mode 100644 tests/test_data/packages/fake_with_local_files/setup.py diff --git a/piptools/resolver.py b/piptools/resolver.py index 14971b0bf..8587d05f1 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -85,6 +85,15 @@ def combine_install_requirements( # Return a sorted, de-duped tuple of extras combined_ireq.extras = tuple(sorted({*combined_ireq.extras, *ireq.extras})) + if ireq.link and combined_ireq.link is None: + combined_ireq.link = ireq.link + + if ireq.local_file_path and combined_ireq.local_file_path is None: + combined_ireq.local_file_path = ireq.local_file_path + + if ireq.original_link and combined_ireq.original_link is None: + combined_ireq.original_link = ireq.original_link + # InstallRequirements objects are assumed to come from only one source, and # so they support only a single comes_from entry. This function breaks this # model. As a workaround, we deterministically choose a single source for diff --git a/tests/test_data/fake-editables.json b/tests/test_data/fake-editables.json index 5075f1e5e..6a65d9d23 100644 --- a/tests/test_data/fake-editables.json +++ b/tests/test_data/fake-editables.json @@ -1,4 +1,5 @@ { "git+git://example.org/django.git#egg=django": [], - "git+https://github.com/celery/billiard#egg=billiard==3.5.9999": [] + "git+https://github.com/celery/billiard#egg=billiard==3.5.9999": [], + "file://localhost//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py": [] } diff --git a/tests/test_data/non_linked_wheel_file/fake_package_a-0.1-py2.py3-none-any.whl b/tests/test_data/non_linked_wheel_file/fake_package_a-0.1-py2.py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..e59cae74b5cfe1595c0f57ebd19fb07fe5b50960 GIT binary patch literal 1631 zcmWIWW@Zs#U|`^2@Y&Mk6{lN&i3iBz0%9#7F3wHN$%#)(%ubC@)HTpE)Jw@MF44`* zOUu`HaSe73@(c*^^!L*%DlXyv`uW>u{_l?;Jz`=AKrxe5wAW_|(5zcPtbyOm+|-i9 zl*E!my{zK=Jm22kd`$)-ZSk!omt1C@TDICsU~e*Ro1I)`v3%x} zbHdVXiGmy1rAi#GwVqhJ#`fuZu0U7ggOf$Yi_O};RD>`&{&RZ1d}m8{LhEjYy=+S# zS=$)49jIMxxtFn;bB$L+*X!&_pG`p;np(b3pK^Kn zY!A{{VRGs0>GR%Keb20J;g(e9{8+?@7OD3F1e3~uF53;n%J^OA>l)(d;uzwXdU_+* zAp?=thkp;%w(oK9dOPE+Z$R+E6`vjRO=sNpn8del?`)PkeDcrjy^hL?epA}nvc2u{NBvx@~W;sh0or6uiI0#EH-FkzRi-2Z=>dG{%;g@?ckL^ z?&>JrH+bfhgZVXE~Dy?gWQroMElfA+u2SvuzWN6v7SV|z3h6ty)TcYnK5 zJL`hEO||%q6Qb*;6q+2rx6zhsR8@}oVue>i^IcfXlM+@9! zW~^SE%P)1p*fOs~BuXIf-mga%)+RQJrXEc3s$S)NT+dba&63?Sqx|&LV1c<638e&00eQuOLCdpv)fTKqb+I{r{s=&KK}`cHU#^_cq8Y!Fe3rH1r&ZFin#QuNz_c5#2!ayn!(A0e%A^IVHfGl?|kY O4G2F0{S(dt;sF5RZ9gUe literal 0 HcmV?d00001 diff --git a/tests/test_data/packages/fake_with_local_files/setup.py b/tests/test_data/packages/fake_with_local_files/setup.py new file mode 100644 index 000000000..6dad2e6d0 --- /dev/null +++ b/tests/test_data/packages/fake_with_local_files/setup.py @@ -0,0 +1,12 @@ +from setuptools import setup +import os + +setup( + name="fake with local files as dependencies", + version=0.1, + install_requires=[ + 'fake_package_a @ file://localhost/{}/tests/test_data/non_linked_wheel_file/fake_package_a-0.1-py2.py3-none-any.whl'.format(os.getcwd()) + ], +) + + diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 03f6e9d9b..fcfb83304 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -1,3 +1,4 @@ +import os import pytest from piptools.exceptions import NoCandidateFound @@ -20,6 +21,10 @@ "werkzeug==0.10.4 (from flask==0.10.1)", ], ), + ( + ["fake_package_a @ file://localhost/{}/tests/test_data/packages/fake_with_local_files/setup.py".format(os.getcwd())], + ["fake_package_a@ file://localhost/{}/tests/test_data/packages/fake_with_local_files/setup.py from file://localhost/{}/tests/test_data/packages/fake_with_local_files/setup.py".format(os.getcwd(), os.getcwd())] + ), (["Jinja2", "markupsafe"], ["jinja2==2.7.3", "markupsafe==0.23"]), # We should return a normal release version if prereleases is False (["SQLAlchemy"], ["sqlalchemy==0.9.9"]), From 88e1d3d6b012637d4317e3599b9a3817cfa7f909 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 14 Apr 2021 21:46:28 -0400 Subject: [PATCH 2/8] Fix issues with linter also add the ability to support other operatign systems. --- tests/constants.py | 2 ++ .../packages/fake_with_local_files/setup.py | 9 ++++++--- tests/test_resolver.py | 14 ++++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/tests/constants.py b/tests/constants.py index 169c742cc..62daf3b9b 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -3,3 +3,5 @@ TEST_DATA_PATH = os.path.join(os.path.dirname(__file__), "test_data") MINIMAL_WHEELS_PATH = os.path.join(TEST_DATA_PATH, "minimal_wheels") PACKAGES_PATH = os.path.join(TEST_DATA_PATH, "packages") +NON_LINKED_PACKAGES_PATH = os.path.join(TEST_DATA_PATH, "non_linked_wheel_file") + diff --git a/tests/test_data/packages/fake_with_local_files/setup.py b/tests/test_data/packages/fake_with_local_files/setup.py index 6dad2e6d0..e68b6e4a2 100644 --- a/tests/test_data/packages/fake_with_local_files/setup.py +++ b/tests/test_data/packages/fake_with_local_files/setup.py @@ -1,12 +1,15 @@ -from setuptools import setup import os +from setuptools import setup +from .constants import NON_LINKED_PACKAGES_PATH setup( name="fake with local files as dependencies", version=0.1, install_requires=[ - 'fake_package_a @ file://localhost/{}/tests/test_data/non_linked_wheel_file/fake_package_a-0.1-py2.py3-none-any.whl'.format(os.getcwd()) - ], + 'fake_package_a @ file://localhost/{}/fake_package_a-0.1-py2.py3-none-any.whl'.format( + os.path.join(NON_LINKED_PACKAGES_PATH) + ) + ], ) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index fcfb83304..c957facf9 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -3,6 +3,7 @@ from piptools.exceptions import NoCandidateFound from piptools.resolver import RequirementSummary, combine_install_requirements +from .constants import PACKAGES_PATH @pytest.mark.parametrize( @@ -22,8 +23,17 @@ ], ), ( - ["fake_package_a @ file://localhost/{}/tests/test_data/packages/fake_with_local_files/setup.py".format(os.getcwd())], - ["fake_package_a@ file://localhost/{}/tests/test_data/packages/fake_with_local_files/setup.py from file://localhost/{}/tests/test_data/packages/fake_with_local_files/setup.py".format(os.getcwd(), os.getcwd())] + [ + "fake_package_a @ file://localhost/{}/fake_with_local_files/setup.py".format( + os.path.join(PACKAGES_PATH) + ) + ], + [ + "fake_package_a@ file://localhost/{}/fake_with_local_files/setup.py "\ + "from file://localhost/{}/fake_with_local_files/setup.py".format( + os.path.join(PACKAGES_PATH), os.path.join(PACKAGES_PATH) + ) + ] ), (["Jinja2", "markupsafe"], ["jinja2==2.7.3", "markupsafe==0.23"]), # We should return a normal release version if prereleases is False From a375dbc9f19e6b3860810de6dd1a4268f7a97c1f Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 14 Apr 2021 22:43:26 -0400 Subject: [PATCH 3/8] Fixing issues with using multiple operating systems --- tests/conftest.py | 4 +++- tests/test_data/fake-editables.json | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3ea43ff24..458a0996a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,7 @@ make_install_requirement, ) -from .constants import MINIMAL_WHEELS_PATH +from .constants import MINIMAL_WHEELS_PATH, PACKAGES_PATH from .utils import looks_like_ci @@ -43,6 +43,8 @@ def __init__(self): with open("tests/test_data/fake-editables.json") as f: self.editables = json.load(f) + self.editables[eval(self.editables['func'])] = [] + print(self.editables) def get_hashes(self, ireq): # Some fake hashes diff --git a/tests/test_data/fake-editables.json b/tests/test_data/fake-editables.json index 6a65d9d23..4fad95278 100644 --- a/tests/test_data/fake-editables.json +++ b/tests/test_data/fake-editables.json @@ -1,5 +1,5 @@ { "git+git://example.org/django.git#egg=django": [], - "git+https://github.com/celery/billiard#egg=billiard==3.5.9999": [], - "file://localhost//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py": [] + "git+https://github.com/celery/billiard#egg=billiard==3.5.9999": [], + "func" : "\"file://localhost/{}/fake_with_local_files/setup.py\".format(os.path.join(PACKAGES_PATH))" } From b6db3f9bc9932ea4824026a0ad3663242e926769 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 14 Apr 2021 23:25:22 -0400 Subject: [PATCH 4/8] Fixing linter issues as well as issues with the newer versio of python --- tests/conftest.py | 7 +++++-- tests/constants.py | 1 - tests/test_data/fake-editables.json | 2 +- .../test_data/packages/fake_with_local_files/setup.py | 6 +++--- tests/test_resolver.py | 10 ++++++---- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 458a0996a..a0b87cc29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,8 +43,11 @@ def __init__(self): with open("tests/test_data/fake-editables.json") as f: self.editables = json.load(f) - self.editables[eval(self.editables['func'])] = [] - print(self.editables) + self.editables[ + self.editables["fake_with_local_files"].format( + os.path.join(PACKAGES_PATH) + ) + ] = [] def get_hashes(self, ireq): # Some fake hashes diff --git a/tests/constants.py b/tests/constants.py index 62daf3b9b..fa302be95 100644 --- a/tests/constants.py +++ b/tests/constants.py @@ -4,4 +4,3 @@ MINIMAL_WHEELS_PATH = os.path.join(TEST_DATA_PATH, "minimal_wheels") PACKAGES_PATH = os.path.join(TEST_DATA_PATH, "packages") NON_LINKED_PACKAGES_PATH = os.path.join(TEST_DATA_PATH, "non_linked_wheel_file") - diff --git a/tests/test_data/fake-editables.json b/tests/test_data/fake-editables.json index 4fad95278..36937204f 100644 --- a/tests/test_data/fake-editables.json +++ b/tests/test_data/fake-editables.json @@ -1,5 +1,5 @@ { "git+git://example.org/django.git#egg=django": [], "git+https://github.com/celery/billiard#egg=billiard==3.5.9999": [], - "func" : "\"file://localhost/{}/fake_with_local_files/setup.py\".format(os.path.join(PACKAGES_PATH))" + "fake_with_local_files": "file://localhost/{}/fake_with_local_files/setup.py" } diff --git a/tests/test_data/packages/fake_with_local_files/setup.py b/tests/test_data/packages/fake_with_local_files/setup.py index e68b6e4a2..41c4eac46 100644 --- a/tests/test_data/packages/fake_with_local_files/setup.py +++ b/tests/test_data/packages/fake_with_local_files/setup.py @@ -1,15 +1,15 @@ import os + from setuptools import setup + from .constants import NON_LINKED_PACKAGES_PATH setup( name="fake with local files as dependencies", version=0.1, install_requires=[ - 'fake_package_a @ file://localhost/{}/fake_package_a-0.1-py2.py3-none-any.whl'.format( + "fake_package_a @ file://localhost/{}/fake_package_a-0.1-py2.py3-none-any.whl".format( os.path.join(NON_LINKED_PACKAGES_PATH) ) ], ) - - diff --git a/tests/test_resolver.py b/tests/test_resolver.py index c957facf9..7dda6276d 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -1,8 +1,10 @@ import os + import pytest from piptools.exceptions import NoCandidateFound from piptools.resolver import RequirementSummary, combine_install_requirements + from .constants import PACKAGES_PATH @@ -27,13 +29,13 @@ "fake_package_a @ file://localhost/{}/fake_with_local_files/setup.py".format( os.path.join(PACKAGES_PATH) ) - ], + ], [ - "fake_package_a@ file://localhost/{}/fake_with_local_files/setup.py "\ - "from file://localhost/{}/fake_with_local_files/setup.py".format( + "fake_package_a@ file://localhost/{}/fake_with_local_files/setup.py " + "from file://localhost/{}/fake_with_local_files/setup.py".format( os.path.join(PACKAGES_PATH), os.path.join(PACKAGES_PATH) ) - ] + ], ), (["Jinja2", "markupsafe"], ["jinja2==2.7.3", "markupsafe==0.23"]), # We should return a normal release version if prereleases is False From 16470ee115392c71b09fbad7d83d92cb3cd62cdd Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 15 Apr 2021 21:38:48 -0400 Subject: [PATCH 5/8] This commit fixes test so that they have a hudered percent code coverage --- tests/test_resolver.py | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 7dda6276d..cc89c6c12 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -24,19 +24,6 @@ "werkzeug==0.10.4 (from flask==0.10.1)", ], ), - ( - [ - "fake_package_a @ file://localhost/{}/fake_with_local_files/setup.py".format( - os.path.join(PACKAGES_PATH) - ) - ], - [ - "fake_package_a@ file://localhost/{}/fake_with_local_files/setup.py " - "from file://localhost/{}/fake_with_local_files/setup.py".format( - os.path.join(PACKAGES_PATH), os.path.join(PACKAGES_PATH) - ) - ], - ), (["Jinja2", "markupsafe"], ["jinja2==2.7.3", "markupsafe==0.23"]), # We should return a normal release version if prereleases is False (["SQLAlchemy"], ["sqlalchemy==0.9.9"]), @@ -316,6 +303,35 @@ def test_combine_install_requirements(repository, from_line): assert str(combined_all.req.specifier) == "<3.2,==3.1.1,>3.0" +def test_combine_install_requirements_with_local_files(repository, from_line): + fake_package = from_line( + "fake_package_a @ file://localhost/{}/fake_with_local_files/setup.py".format( + os.path.join(PACKAGES_PATH) + ), + comes_from="-r requirements.in", + ) + fake_package_name = from_line( + "fake_package_a==1.0.0", comes_from=from_line("fake_package_a") + ) + + combined_all = combine_install_requirements( + repository, [fake_package, fake_package_name] + ) + assert str(combined_all.req.specifier) == "==1.0.0" + assert ( + str(combined_all.link) + == "file://localhost//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py" + ) + assert ( + str(combined_all.local_file_path) + == "//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py" + ) + assert ( + str(combined_all.original_link) + == "file://localhost//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py" + ) + + def test_compile_failure_shows_provenance(resolver, from_line): """ Provenance of conflicting dependencies should be printed on failure. From a654cf9bf69b3a204ae71a3d7f56b76d4cf2eb52 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 15 Apr 2021 21:52:49 -0400 Subject: [PATCH 6/8] make tests work with multiple OS --- tests/test_resolver.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index cc89c6c12..c48b976a6 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -318,17 +318,19 @@ def test_combine_install_requirements_with_local_files(repository, from_line): repository, [fake_package, fake_package_name] ) assert str(combined_all.req.specifier) == "==1.0.0" - assert ( - str(combined_all.link) - == "file://localhost//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py" + assert str( + combined_all.link + ) == "file://localhost/{}/fake_with_local_files/setup.py".format( + os.path.join(PACKAGES_PATH) ) assert ( str(combined_all.local_file_path) - == "//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py" + == f"/{os.path.join(PACKAGES_PATH)}fake_with_local_files/setup.py" ) - assert ( - str(combined_all.original_link) - == "file://localhost//src/piptools/tests/test_data/packages/fake_with_local_files/setup.py" + assert str( + combined_all.original_link + ) == "file://localhost/{}/fake_with_local_files/setup.py".format( + os.path.join(PACKAGES_PATH) ) From 9493624b4f0f78bd58f5c6ee17b3d55fbffb27af Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 15 Apr 2021 21:57:55 -0400 Subject: [PATCH 7/8] Fix small mistake with file path in test --- tests/test_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index c48b976a6..b5b34d863 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -325,7 +325,7 @@ def test_combine_install_requirements_with_local_files(repository, from_line): ) assert ( str(combined_all.local_file_path) - == f"/{os.path.join(PACKAGES_PATH)}fake_with_local_files/setup.py" + == f"/{os.path.join(PACKAGES_PATH)}/fake_with_local_files/setup.py" ) assert str( combined_all.original_link From 274cf5f5fb244ddece66e74589c3b37f47cebfd9 Mon Sep 17 00:00:00 2001 From: Andrew Date: Thu, 15 Apr 2021 22:45:24 -0400 Subject: [PATCH 8/8] Fix small mistake with file path in test --- tests/test_resolver.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index b5b34d863..5a46b5e24 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -305,9 +305,8 @@ def test_combine_install_requirements(repository, from_line): def test_combine_install_requirements_with_local_files(repository, from_line): fake_package = from_line( - "fake_package_a @ file://localhost/{}/fake_with_local_files/setup.py".format( - os.path.join(PACKAGES_PATH) - ), + "fake_package_a @ file://localhost/" + + os.path.join(PACKAGES_PATH, "fake_with_local_files", "setup.py"), comes_from="-r requirements.in", ) fake_package_name = from_line( @@ -318,20 +317,9 @@ def test_combine_install_requirements_with_local_files(repository, from_line): repository, [fake_package, fake_package_name] ) assert str(combined_all.req.specifier) == "==1.0.0" - assert str( - combined_all.link - ) == "file://localhost/{}/fake_with_local_files/setup.py".format( - os.path.join(PACKAGES_PATH) - ) - assert ( - str(combined_all.local_file_path) - == f"/{os.path.join(PACKAGES_PATH)}/fake_with_local_files/setup.py" - ) - assert str( - combined_all.original_link - ) == "file://localhost/{}/fake_with_local_files/setup.py".format( - os.path.join(PACKAGES_PATH) - ) + assert str(combined_all.link) == str(fake_package.link) + assert str(combined_all.local_file_path) == str(fake_package.local_file_path) + assert str(combined_all.original_link) == str(fake_package.original_link) def test_compile_failure_shows_provenance(resolver, from_line):