From a3c793618f9487ca99de203ff677227ef42b4124 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 18 Jul 2024 11:19:22 +0000 Subject: [PATCH 01/12] Add support for SHA256 hashes in explicit files --- conda/cli/main_list.py | 27 ++++++++++++++++++++++----- conda/misc.py | 17 ++++++++++++----- tests/test_export.py | 21 +++++++++++---------- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/conda/cli/main_list.py b/conda/cli/main_list.py index 37814addb87..0c481e9d897 100644 --- a/conda/cli/main_list.py +++ b/conda/cli/main_list.py @@ -94,6 +94,11 @@ def configure_parser(sub_parsers: _SubParsersAction, **kwargs) -> ArgumentParser action="store_true", help="Add MD5 hashsum when using --explicit.", ) + p.add_argument( + "--sha256", + action="store_true", + help="Add SHA256 hashsum when using --explicit.", + ) p.add_argument( "-e", "--export", @@ -244,12 +249,13 @@ def print_packages( return exitcode -def print_explicit(prefix, add_md5=False, remove_auth=True): +def print_explicit(prefix, add_md5=False, remove_auth=True, add_sha256=False): from ..base.constants import UNKNOWN_CHANNEL from ..base.context import context from ..common import url as common_url from ..core.prefix_data import PrefixData - + if add_md5 and add_sha256: + raise ValueError("Only one of add_md5 and add_sha256 can be chosen") if not isdir(prefix): from ..exceptions import EnvironmentLocationNotFound @@ -263,8 +269,14 @@ def print_explicit(prefix, add_md5=False, remove_auth=True): continue if remove_auth: url = common_url.remove_auth(common_url.split_anaconda_token(url)[0]) - md5 = prefix_record.get("md5") - print(url + (f"#{md5}" if add_md5 and md5 else "")) + if add_md5: + md5 = prefix_record.get("md5") + print(url + (f"#{md5}" if md5 else "")) + elif add_sha256: + sha256 = prefix_record.get("sha256") + print(url + (f"#{sha256}" if sha256 else "")) + else: + print(url) def execute(args: Namespace, parser: ArgumentParser) -> int: @@ -279,6 +291,11 @@ def execute(args: Namespace, parser: ArgumentParser) -> int: raise EnvironmentLocationNotFound(prefix) + if args.md5 and args.sha256: + from ..exceptions import ArgumentError + + raise ArgumentError("Only one of --md5 and --sha256 can be specified at the same time") + regex = args.regex if args.full_name: regex = rf"^{regex}$" @@ -297,7 +314,7 @@ def execute(args: Namespace, parser: ArgumentParser) -> int: return 0 if args.explicit: - print_explicit(prefix, args.md5, args.remove_auth) + print_explicit(prefix, args.md5, args.remove_auth, args.sha256) return 0 if args.canonical: diff --git a/conda/misc.py b/conda/misc.py index 94ad035911d..6f6ff26a874 100644 --- a/conda/misc.py +++ b/conda/misc.py @@ -50,7 +50,10 @@ def conda_installed_files(prefix, exclude_self_build=False): url_pat = re.compile( r"(?:(?P.+)(?:[/\\]))?" r"(?P[^/\\#]+(?:\.tar\.bz2|\.conda))" - r"(:?#(?P[0-9a-f]{32}))?$" + r"(:?#(" + r"(?P[0-9a-f]{32})" + r"|(?P[0-9a-f]{64})" + r"))?$" ) @@ -81,11 +84,15 @@ def explicit( m = url_pat.match(spec) if m is None: raise ParseError(f"Could not parse explicit URL: {spec}") - url_p, fn, md5sum = m.group("url_p"), m.group("fn"), m.group("md5") + url_p, fn = m.group("url_p"), m.group("fn") url = join_url(url_p, fn) - # url_p is everything but the tarball_basename and the md5sum - - fetch_specs.append(MatchSpec(url, md5=md5sum) if md5sum else MatchSpec(url)) + # url_p is everything but the tarball_basename and the checksum + checksums = {} + if md5 := m.group("md5"): + checksums["md5"] = md5 + if sha256 := m.group("sha256"): + checksums["sha256"] = sha256 + fetch_specs.append(MatchSpec(url, **checksums)) if context.dry_run: raise DryRunExit() diff --git a/tests/test_export.py b/tests/test_export.py index 479e1021a2d..c1c90626226 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -16,7 +16,6 @@ def test_export( conda_cli: CondaCLIFixture, path_factory: PathFactoryFixture, monkeypatch: MonkeyPatch, - request, ): """Test that `conda list --export` output can be used to create a similar environment.""" monkeypatch.setenv("CONDA_CHANNELS", "defaults") @@ -39,12 +38,14 @@ def test_export( assert output == output2 +# Using --quiet here as a no-op flag for test simplicity +@pytest.mark.parametrize("checksum_flag", ("--quiet", "--md5", "--sha256")) @pytest.mark.integration def test_explicit( tmp_env: TmpEnvFixture, conda_cli: CondaCLIFixture, path_factory: PathFactoryFixture, - request, + checksum_flag: str, ): """Test that `conda list --explicit` output can be used to recreate an identical environment.""" # use "cheap" packages with no dependencies @@ -52,14 +53,14 @@ def test_explicit( assert package_is_installed(prefix, "pkgs/main::zlib") assert package_is_installed(prefix, "conda-forge::ca-certificates") - output, _, _ = conda_cli("list", "--prefix", prefix, "--explicit") + output, _, _ = conda_cli("list", "--prefix", prefix, "--explicit", checksum_flag) - env_txt = path_factory(suffix=".txt") - env_txt.write_text(output) + env_txt = path_factory(suffix=".txt") + env_txt.write_text(output) - with tmp_env("--file", env_txt) as prefix2: - assert package_is_installed(prefix2, "pkgs/main::zlib") - assert package_is_installed(prefix2, "conda-forge::ca-certificates") + with tmp_env("--file", env_txt) as prefix2: + assert package_is_installed(prefix2, "pkgs/main::zlib") + assert package_is_installed(prefix2, "conda-forge::ca-certificates") - output2, _, _ = conda_cli("list", "--prefix", prefix2, "--explicit") - assert output == output2 + output2, _, _ = conda_cli("list", "--prefix", prefix2, "--explicit", checksum_flag) + assert output == output2 From 3628763f414f66daee17954ee3020ad0a2872dfc Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 18 Jul 2024 11:30:41 +0000 Subject: [PATCH 02/12] add created-by header --- conda/cli/main_list.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda/cli/main_list.py b/conda/cli/main_list.py index 0c481e9d897..86cf2d7d552 100644 --- a/conda/cli/main_list.py +++ b/conda/cli/main_list.py @@ -10,6 +10,8 @@ from argparse import ArgumentParser, Namespace, _SubParsersAction from os.path import isdir, isfile +from .. import __version__ + log = logging.getLogger(__name__) @@ -143,6 +145,7 @@ def print_export_header(subdir): print("# This file may be used to create an environment using:") print("# $ conda create --name --file ") print(f"# platform: {subdir}") + print(f"# created-by: conda {__version__}") def get_packages(installed, regex): From a7c9d0a1cdcf3015b1da8050f0da0d8767f94240 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 18 Jul 2024 11:35:53 +0000 Subject: [PATCH 03/12] pre-commit --- conda/cli/main_list.py | 5 ++++- conda/misc.py | 6 +++--- tests/test_export.py | 8 ++++++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/conda/cli/main_list.py b/conda/cli/main_list.py index 86cf2d7d552..760ab2b8e58 100644 --- a/conda/cli/main_list.py +++ b/conda/cli/main_list.py @@ -257,6 +257,7 @@ def print_explicit(prefix, add_md5=False, remove_auth=True, add_sha256=False): from ..base.context import context from ..common import url as common_url from ..core.prefix_data import PrefixData + if add_md5 and add_sha256: raise ValueError("Only one of add_md5 and add_sha256 can be chosen") if not isdir(prefix): @@ -297,7 +298,9 @@ def execute(args: Namespace, parser: ArgumentParser) -> int: if args.md5 and args.sha256: from ..exceptions import ArgumentError - raise ArgumentError("Only one of --md5 and --sha256 can be specified at the same time") + raise ArgumentError( + "Only one of --md5 and --sha256 can be specified at the same time" + ) regex = args.regex if args.full_name: diff --git a/conda/misc.py b/conda/misc.py index 6f6ff26a874..43399ef9559 100644 --- a/conda/misc.py +++ b/conda/misc.py @@ -51,8 +51,8 @@ def conda_installed_files(prefix, exclude_self_build=False): r"(?:(?P.+)(?:[/\\]))?" r"(?P[^/\\#]+(?:\.tar\.bz2|\.conda))" r"(:?#(" - r"(?P[0-9a-f]{32})" - r"|(?P[0-9a-f]{64})" + r"(?P[0-9a-f]{32})" + r"|(?P[0-9a-f]{64})" r"))?$" ) @@ -89,7 +89,7 @@ def explicit( # url_p is everything but the tarball_basename and the checksum checksums = {} if md5 := m.group("md5"): - checksums["md5"] = md5 + checksums["md5"] = md5 if sha256 := m.group("sha256"): checksums["sha256"] = sha256 fetch_specs.append(MatchSpec(url, **checksums)) diff --git a/tests/test_export.py b/tests/test_export.py index c1c90626226..88066b231ae 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -53,7 +53,9 @@ def test_explicit( assert package_is_installed(prefix, "pkgs/main::zlib") assert package_is_installed(prefix, "conda-forge::ca-certificates") - output, _, _ = conda_cli("list", "--prefix", prefix, "--explicit", checksum_flag) + output, _, _ = conda_cli( + "list", "--prefix", prefix, "--explicit", checksum_flag + ) env_txt = path_factory(suffix=".txt") env_txt.write_text(output) @@ -62,5 +64,7 @@ def test_explicit( assert package_is_installed(prefix2, "pkgs/main::zlib") assert package_is_installed(prefix2, "conda-forge::ca-certificates") - output2, _, _ = conda_cli("list", "--prefix", prefix2, "--explicit", checksum_flag) + output2, _, _ = conda_cli( + "list", "--prefix", prefix2, "--explicit", checksum_flag + ) assert output == output2 From 02221d5fd02c780b4f3ef7b689e210310024dedd Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 18 Jul 2024 15:10:10 +0000 Subject: [PATCH 04/12] Support sha256: prefix in the hash --- conda/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/misc.py b/conda/misc.py index 43399ef9559..e0c0d9b3305 100644 --- a/conda/misc.py +++ b/conda/misc.py @@ -52,7 +52,7 @@ def conda_installed_files(prefix, exclude_self_build=False): r"(?P[^/\\#]+(?:\.tar\.bz2|\.conda))" r"(:?#(" r"(?P[0-9a-f]{32})" - r"|(?P[0-9a-f]{64})" + r"|((sha256:)?(?P[0-9a-f]{64}))" r"))?$" ) From 867c7d7c1f30a66746f62a7b0fea9bd92df36761 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 18 Jul 2024 15:31:29 +0000 Subject: [PATCH 05/12] add parsing tests --- conda/misc.py | 20 ++++++++------ tests/test_export.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/conda/misc.py b/conda/misc.py index e0c0d9b3305..ccf607285ed 100644 --- a/conda/misc.py +++ b/conda/misc.py @@ -9,6 +9,7 @@ from collections import defaultdict from logging import getLogger from os.path import abspath, dirname, exists, isdir, isfile, join, relpath +from typing import Iterable from .base.context import context from .common.compat import on_mac, on_win, open @@ -57,13 +58,7 @@ def conda_installed_files(prefix, exclude_self_build=False): ) -def explicit( - specs, prefix, verbose=False, force_extract=True, index_args=None, index=None -): - actions = defaultdict(list) - actions["PREFIX"] = prefix - - fetch_specs = [] +def _match_specs_from_explicit(specs: Iterable[str]) -> Iterable[MatchSpec]: for spec in specs: if spec == "@EXPLICIT": continue @@ -92,7 +87,16 @@ def explicit( checksums["md5"] = md5 if sha256 := m.group("sha256"): checksums["sha256"] = sha256 - fetch_specs.append(MatchSpec(url, **checksums)) + yield MatchSpec(url, **checksums) + + +def explicit( + specs, prefix, verbose=False, force_extract=True, index_args=None, index=None +): + actions = defaultdict(list) + actions["PREFIX"] = prefix + + fetch_specs = list(_match_specs_from_explicit(specs)) if context.dry_run: raise DryRunExit() diff --git a/tests/test_export.py b/tests/test_export.py index 88066b231ae..f89db791dac 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,9 +1,14 @@ # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause +import os +from contextlib import nullcontext + import pytest from pytest import MonkeyPatch from conda.base.context import reset_context +from conda.exceptions import ParseError +from conda.misc import _match_specs_from_explicit from conda.testing import CondaCLIFixture, PathFactoryFixture, TmpEnvFixture from conda.testing.integration import package_is_installed @@ -68,3 +73,60 @@ def test_explicit( "list", "--prefix", prefix2, "--explicit", checksum_flag ) assert output == output2 + +@pytest.mark.parametrize( + "url, checksum, raises", + ( + [ + "https://conda.anaconda.org/conda-forge/noarch/doc8-1.1.1-pyhd8ed1ab_0.conda", + "5e9e17751f19d03c4034246de428582e", + None, + ], + [ + "https://conda.anaconda.org/conda-forge/noarch/conda-24.1.0-pyhd3eb1b0_0.conda", + "2707f68aada792d1cf3a44c51d55b38b0cd65b0c192d2a5f9ef0550dc149a7d3", + None, + ], + [ + "https://conda.anaconda.org/conda-forge/noarch/conda-24.1.0-pyhd3eb1b0_0.conda", + "sha256:2707f68aada792d1cf3a44c51d55b38b0cd65b0c192d2a5f9ef0550dc149a7d3", + None, + ], + [ + "https://conda.anaconda.org/conda-forge/noarch/conda-24.1.0-pyhd3eb1b0_0.conda", + "sha123:2707f68aada792d1cf3a44c51d55b38b0cd65b0c192d2a5f9ef0550dc149a7d3", + ParseError, + ], + [ + "https://conda.anaconda.org/conda-forge/noarch/conda-24.1.0-pyhd3eb1b0_0.conda", + "md5:5e9e17751f19d03c4034246de428582e", # this is not valid syntax; use without 'md5:' + ParseError, + ], + [ + "doc8-1.1.1-pyhd8ed1ab_0.conda", + "5e9e17751f19d03c4034246de428582e", + None, + ], + [ + "../doc8-1.1.1-pyhd8ed1ab_0.conda", + "5e9e17751f19d03c4034246de428582e", + None, + ], + [ + "../doc8-1.1.1-pyhd8ed1ab_0.conda", + "5e9e17751f19d03", + ParseError, + ], + ) +) +def test_explicit_parser(url: str, checksum: str, raises: Exception | None): + lines = [url + (f"#{checksum}" if checksum else "")] + with pytest.raises(raises) if raises else nullcontext(): + specs = list(_match_specs_from_explicit(lines)) + if raises: + return + + assert len(specs) == 1 + spec = specs[0] + assert spec.get("url").split("/")[-1] == url.split("/")[-1] + assert checksum.rsplit(":", 1)[-1] in (spec.get("md5"), spec.get("sha256")) \ No newline at end of file From d7b5c5b192b150e8477ae3577e7b510160d7ded5 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 19 Jul 2024 08:02:56 +0000 Subject: [PATCH 06/12] future annotations --- tests/test_export.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_export.py b/tests/test_export.py index f89db791dac..58e0f6d5e55 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -1,6 +1,7 @@ # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause -import os +from __future__ import annotations + from contextlib import nullcontext import pytest From 2b75aacd68a4ba99b91316512abb232d59257662 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 19 Jul 2024 16:31:08 +0200 Subject: [PATCH 07/12] pre-commit --- tests/test_export.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_export.py b/tests/test_export.py index 58e0f6d5e55..e2911642a21 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -75,6 +75,7 @@ def test_explicit( ) assert output == output2 + @pytest.mark.parametrize( "url, checksum, raises", ( @@ -118,7 +119,7 @@ def test_explicit( "5e9e17751f19d03", ParseError, ], - ) + ), ) def test_explicit_parser(url: str, checksum: str, raises: Exception | None): lines = [url + (f"#{checksum}" if checksum else "")] @@ -130,4 +131,4 @@ def test_explicit_parser(url: str, checksum: str, raises: Exception | None): assert len(specs) == 1 spec = specs[0] assert spec.get("url").split("/")[-1] == url.split("/")[-1] - assert checksum.rsplit(":", 1)[-1] in (spec.get("md5"), spec.get("sha256")) \ No newline at end of file + assert checksum.rsplit(":", 1)[-1] in (spec.get("md5"), spec.get("sha256")) From c7d91321e81ca2f5dfef5faf22f60bcf861701ee Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 23 Jul 2024 10:51:28 +0200 Subject: [PATCH 08/12] add news --- news/14048-explicit-sha256 | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 news/14048-explicit-sha256 diff --git a/news/14048-explicit-sha256 b/news/14048-explicit-sha256 new file mode 100644 index 00000000000..6aa19c19077 --- /dev/null +++ b/news/14048-explicit-sha256 @@ -0,0 +1,21 @@ +### Enhancements + +* Add `--sha256` flag to `conda list --explicit` so it lists URLs with a SHA256 hash instead of MD5. (#14048) +* Add support for SHA256 hashes in `@EXPLICIT` text files consumed by `conda install|create`. (#14048) +* Report `conda` version used to generate a `@EXPLICIT` text file. (#14048) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From bdcc774fb1b9603e80253b55a15790f2367b2cfe Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 23 Jul 2024 10:57:18 +0200 Subject: [PATCH 09/12] pre-commit --- tests/test_export.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_export.py b/tests/test_export.py index e2911642a21..32b0cfb4bd2 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -3,16 +3,20 @@ from __future__ import annotations from contextlib import nullcontext +from typing import TYPE_CHECKING import pytest -from pytest import MonkeyPatch from conda.base.context import reset_context from conda.exceptions import ParseError from conda.misc import _match_specs_from_explicit -from conda.testing import CondaCLIFixture, PathFactoryFixture, TmpEnvFixture from conda.testing.integration import package_is_installed +if TYPE_CHECKING: + from pytest import MonkeyPatch + + from conda.testing import CondaCLIFixture, PathFactoryFixture, TmpEnvFixture + pytestmark = pytest.mark.usefixtures("parametrized_solver_fixture") From dbc1ba209aea277fa161a4136641de955575cf29 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 24 Jul 2024 10:41:00 +0200 Subject: [PATCH 10/12] Amend news --- news/14048-explicit-sha256 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/news/14048-explicit-sha256 b/news/14048-explicit-sha256 index 6aa19c19077..85731217a89 100644 --- a/news/14048-explicit-sha256 +++ b/news/14048-explicit-sha256 @@ -1,7 +1,6 @@ ### Enhancements -* Add `--sha256` flag to `conda list --explicit` so it lists URLs with a SHA256 hash instead of MD5. (#14048) -* Add support for SHA256 hashes in `@EXPLICIT` text files consumed by `conda install|create`. (#14048) +* Add `--sha256` flag to `conda list --explicit` so it lists URLs with a SHA256 hash instead of MD5 and make `conda install|create` compatible with these inputs. (#2903, #7882 via #14048) * Report `conda` version used to generate a `@EXPLICIT` text file. (#14048) ### Bug fixes From f7fc1b1f59a56a604114251426ccb5c73d7d8f44 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 27 Aug 2024 09:46:57 +0200 Subject: [PATCH 11/12] Apply suggestions from code review Co-authored-by: Daniel Holth --- conda/cli/main_list.py | 5 ++++- conda/misc.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conda/cli/main_list.py b/conda/cli/main_list.py index 760ab2b8e58..10196bcf6c3 100644 --- a/conda/cli/main_list.py +++ b/conda/cli/main_list.py @@ -278,7 +278,10 @@ def print_explicit(prefix, add_md5=False, remove_auth=True, add_sha256=False): print(url + (f"#{md5}" if md5 else "")) elif add_sha256: sha256 = prefix_record.get("sha256") - print(url + (f"#{sha256}" if sha256 else "")) + if add_md5 or add_sha256: + hash_key = "md5" if add_md5 else "sha256" + hash_value = prefix_record.get(hash_key) + print(url + (f"#{hash_value}" if hash_value else "")) else: print(url) diff --git a/conda/misc.py b/conda/misc.py index ccf607285ed..f850415c8f2 100644 --- a/conda/misc.py +++ b/conda/misc.py @@ -51,7 +51,7 @@ def conda_installed_files(prefix, exclude_self_build=False): url_pat = re.compile( r"(?:(?P.+)(?:[/\\]))?" r"(?P[^/\\#]+(?:\.tar\.bz2|\.conda))" - r"(:?#(" + r"(?:#(" r"(?P[0-9a-f]{32})" r"|((sha256:)?(?P[0-9a-f]{64}))" r"))?$" From 523d29b7990bee2954876306a9cf1419d118aa53 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 27 Aug 2024 09:47:55 +0200 Subject: [PATCH 12/12] Update conda/cli/main_list.py --- conda/cli/main_list.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/conda/cli/main_list.py b/conda/cli/main_list.py index 10196bcf6c3..7508fba6984 100644 --- a/conda/cli/main_list.py +++ b/conda/cli/main_list.py @@ -273,11 +273,6 @@ def print_explicit(prefix, add_md5=False, remove_auth=True, add_sha256=False): continue if remove_auth: url = common_url.remove_auth(common_url.split_anaconda_token(url)[0]) - if add_md5: - md5 = prefix_record.get("md5") - print(url + (f"#{md5}" if md5 else "")) - elif add_sha256: - sha256 = prefix_record.get("sha256") if add_md5 or add_sha256: hash_key = "md5" if add_md5 else "sha256" hash_value = prefix_record.get(hash_key)