From 48c7279e6d8aba6a789926c21be43002787c0603 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 9 May 2024 09:25:22 +0200 Subject: [PATCH 01/22] add list extensions --- conda_pypi/cli/__init__.py | 1 + conda_pypi/cli/install.py | 3 +++ conda_pypi/cli/list.py | 42 +++++++++++++++++++++++++++++++ conda_pypi/{cli.py => cli/pip.py} | 8 +++--- conda_pypi/plugin.py | 16 +++++++++--- 5 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 conda_pypi/cli/__init__.py create mode 100644 conda_pypi/cli/install.py create mode 100644 conda_pypi/cli/list.py rename conda_pypi/{cli.py => cli/pip.py} (97%) diff --git a/conda_pypi/cli/__init__.py b/conda_pypi/cli/__init__.py new file mode 100644 index 0000000..3767cfa --- /dev/null +++ b/conda_pypi/cli/__init__.py @@ -0,0 +1 @@ +from . import install, list, pip diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py new file mode 100644 index 0000000..22f300a --- /dev/null +++ b/conda_pypi/cli/install.py @@ -0,0 +1,3 @@ +def post_command(command: str): + if command not in ("install", "create"): + return diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py new file mode 100644 index 0000000..db6aad1 --- /dev/null +++ b/conda_pypi/cli/list.py @@ -0,0 +1,42 @@ +import json +import sys +from pathlib import Path + +from conda.base.context import context +from conda.core.prefix_data import PrefixData +from conda.models.enums import PackageType + +def post_command(command: str): + if command != "list": + return + if "--explicit" not in sys.argv: + return + PrefixData._cache_.clear() + pd = PrefixData(context.target_prefix, pip_interop_enabled=True) + pd.load() + to_print = [] + for record in pd.iter_records(): + if record.package_type != PackageType.VIRTUAL_PYTHON_WHEEL: + continue + ignore = False + for path in record.files: + path = Path(context.target_prefix, path) + if "__editable__" in path.stem: + ignore = True + break + if path.name == "direct_url.json": + data = json.loads(path.read_text()) + if data.get("dir_info", {}).get("editable"): + ignore = True + break + if ignore: + continue + + if record.url: + to_print.append(f"# pypi: {record.url}") + else: + to_print.append(f"# pypi: {record.name}=={record.version}") + + if to_print: + print("# Following lines added by conda-pypi") + print(*to_print, sep="\n") diff --git a/conda_pypi/cli.py b/conda_pypi/cli/pip.py similarity index 97% rename from conda_pypi/cli.py rename to conda_pypi/cli/pip.py index 5c6f940..941c100 100644 --- a/conda_pypi/cli.py +++ b/conda_pypi/cli/pip.py @@ -20,7 +20,7 @@ def configure_parser(parser: argparse.ArgumentParser): - from .dependencies import BACKENDS + from ..dependencies import BACKENDS add_parser_help(parser) add_parser_prefix(parser) @@ -69,14 +69,14 @@ def configure_parser(parser: argparse.ArgumentParser): def execute(args: argparse.Namespace) -> int: from conda.common.io import Spinner from conda.models.match_spec import MatchSpec - from .dependencies import analyze_dependencies - from .main import ( + from ..dependencies import analyze_dependencies + from ..main import ( validate_target_env, ensure_externally_managed, run_conda_install, run_pip_install, ) - from .utils import get_prefix + from ..utils import get_prefix prefix = get_prefix(args.prefix, args.name) packages_not_installed = validate_target_env(prefix, args.packages) diff --git a/conda_pypi/plugin.py b/conda_pypi/plugin.py index a95b6fb..38dc4c9 100644 --- a/conda_pypi/plugin.py +++ b/conda_pypi/plugin.py @@ -1,6 +1,6 @@ from conda import plugins -from .cli import configure_parser, execute +from . import cli from .main import ensure_target_env_has_externally_managed @@ -9,8 +9,8 @@ def conda_subcommands(): yield plugins.CondaSubcommand( name="pip", summary="Run pip commands within conda environments in a safer way", - action=execute, - configure_parser=configure_parser, + action=cli.pip.execute, + configure_parser=cli.pip.configure_parser, ) @@ -21,3 +21,13 @@ def conda_post_commands(): action=ensure_target_env_has_externally_managed, run_for={"install", "create", "update", "remove"}, ) + yield plugins.CondaPostCommand( + name="conda-pypi-post-list", + action=cli.list.post_command, + run_for={"list"}, + ) + yield plugins.CondaPostCommand( + name="conda-pypi-post-install-create", + action=cli.install.post_command, + run_for={"install", "create"}, + ) From bc1d8c4d85cce63b86dd67933770547602958167 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 17 May 2024 16:21:43 +0200 Subject: [PATCH 02/22] add more details to the explicit list --- conda_pypi/cli/list.py | 32 ++----------------- conda_pypi/main.py | 72 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 30 deletions(-) diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index db6aad1..e916aeb 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -1,41 +1,15 @@ -import json import sys -from pathlib import Path from conda.base.context import context -from conda.core.prefix_data import PrefixData -from conda.models.enums import PackageType + +from ..main import pypi_lines_for_explicit_lockfile def post_command(command: str): if command != "list": return if "--explicit" not in sys.argv: return - PrefixData._cache_.clear() - pd = PrefixData(context.target_prefix, pip_interop_enabled=True) - pd.load() - to_print = [] - for record in pd.iter_records(): - if record.package_type != PackageType.VIRTUAL_PYTHON_WHEEL: - continue - ignore = False - for path in record.files: - path = Path(context.target_prefix, path) - if "__editable__" in path.stem: - ignore = True - break - if path.name == "direct_url.json": - data = json.loads(path.read_text()) - if data.get("dir_info", {}).get("editable"): - ignore = True - break - if ignore: - continue - - if record.url: - to_print.append(f"# pypi: {record.url}") - else: - to_print.append(f"# pypi: {record.name}=={record.version}") + to_print = pypi_lines_for_explicit_lockfile(context.target_prefix) if to_print: print("# Following lines added by conda-pypi") diff --git a/conda_pypi/main.py b/conda_pypi/main.py index a2a2ea0..327d090 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os from logging import getLogger from pathlib import Path @@ -11,9 +12,11 @@ except ImportError: from importlib_resources import files as importlib_files -from conda.history import History + from conda.base.context import context from conda.core.prefix_data import PrefixData +from conda.models.enums import PackageType +from conda.history import History from conda.cli.python_api import run_command from conda.exceptions import CondaError, CondaSystemExit from conda.models.match_spec import MatchSpec @@ -168,3 +171,70 @@ def ensure_target_env_has_externally_managed(command: str): path.unlink() else: raise ValueError(f"command {command} not recognized.") + + +def pypi_lines_for_explicit_lockfile(prefix: Path | str) -> list[str]: + PrefixData._cache_.clear() + pd = PrefixData(str(prefix), pip_interop_enabled=True) + pd.load() + lines = [] + python_record = list(pd.query("python")) + assert len(python_record) == 1 + python_record = python_record[0] + python_details = {"version": ".".join(python_record.version.split(".")[:3])} + if "pypy" in python_record.build: + python_details["implementation"] = "pp" + else: + python_details["implementation"] = "cp" + for record in pd.iter_records(): + if record.package_type != PackageType.VIRTUAL_PYTHON_WHEEL: + continue + ignore = False + wheel = {} + for path in record.files: + path = Path(context.target_prefix, path) + if "__editable__" in path.stem: + ignore = True + break + if path.name == "direct_url.json": + data = json.loads(path.read_text()) + if data.get("dir_info", {}).get("editable"): + ignore = True + break + if path.name == "WHEEL": + for line in path.read_text().splitlines(): + line = line.strip() + if ":" not in line: + continue + key, value = line.split(":", 1) + if key == "Tag": + wheel.setdefault(key, []).append(value.strip()) + else: + wheel[key] = value.strip() + if ignore: + continue + if record.url: + lines.append(f"# pypi: {record.url}") + else: + seen = {"abi": set(), "platform": set()} + lines.append(f"# pypi: {record.name}=={record.version}") + if wheel and (wheel_tag := wheel.get('Tag')): + for i, tag in enumerate(wheel_tag): + python_tag, abi_tag, platform_tag = tag.split("-", 2) + if i == 0: # only once + lines[-1] += f" --python-version {python_tag[2:]}" + lines[-1] += f" --implementation {python_tag[:2]}" + if abi_tag not in seen["abi"]: + lines[-1] += f" --abi {abi_tag}" + seen["abi"].add(abi_tag) + if platform_tag not in seen["platform"]: + lines[-1] += f" --platform {platform_tag}" + seen["platform"].add(platform_tag) + else: + lines[-1] += f" --python-version {python_details['version']}" + lines[-1] += f" --implementation {python_details['implementation']}" + # Here we could try to run a --dry-run --report some.json to get the resolved URL + # but it's not guaranteed we get the exact same source so for now we defer to install + # time + + return lines From 3dd66577feefa9cbc68fa12f3fb2eca7876011e2 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 17 May 2024 18:33:04 +0200 Subject: [PATCH 03/22] implement install from '# pypi:' lockfiles --- conda_pypi/cli/install.py | 58 ++++++++++++++++++ conda_pypi/cli/pip.py | 6 +- conda_pypi/dependencies/pip.py | 46 +------------- conda_pypi/main.py | 106 ++++++++++++++++++++++++++------- conda_pypi/utils.py | 14 ++++- 5 files changed, 157 insertions(+), 73 deletions(-) diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index 22f300a..b125cca 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -1,3 +1,61 @@ +from __future__ import annotations + +import shlex +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +from conda.base.context import context +from conda.common.io import Spinner + +from ..main import run_pip_install, pip_install_download_info + +if TYPE_CHECKING: + from typing import Iterable + + def post_command(command: str): if command not in ("install", "create"): return + + installed = [] + pypi_lines = pypi_lines_from_sys_argv() + + with Spinner( + f"Installing PyPI packages ({len(pypi_lines)})", + enabled=not context.quiet, + json=context.json, + ): + for args in pypi_lines: + args = shlex.split(args) + dl_info = pip_install_download_info(args) + run_pip_install( + context.target_prefix, + args=[dl_info["url"]], + dry_run=context.dry_run, + quiet=context.quiet, + verbosity=context.verbosity, + force_reinstall=context.force_reinstall, + yes=context.always_yes, + check=True, + ) + installed.append(args[0]) + print("Successfully installed PyPI packages:", *installed) + return 0 + + +def pypi_lines_from_sys_argv(argv: Iterable[str] | None = None) -> list[str]: + argv = argv or sys.argv + if "--file" not in argv: + return [] + pypi_lines = [] + pypi_prefix = "# pypi: " + pypi_prefix_len = len(pypi_prefix) + for i, arg in enumerate(argv): + if arg == "--file": + pypi_lines += [ + line[pypi_prefix_len:] + for line in Path(argv[i + 1]).read_text().splitlines() + if line.strip().startswith(pypi_prefix) + ] + return pypi_lines diff --git a/conda_pypi/cli/pip.py b/conda_pypi/cli/pip.py index 941c100..507e203 100644 --- a/conda_pypi/cli/pip.py +++ b/conda_pypi/cli/pip.py @@ -150,7 +150,7 @@ def execute(args: argparse.Namespace) -> int: if pypi_specs: if not args.quiet or not args.json: print("Running pip install...") - retcode = run_pip_install( + process = run_pip_install( prefix, pypi_specs, dry_run=args.dry_run, @@ -159,8 +159,8 @@ def execute(args: argparse.Namespace) -> int: force_reinstall=args.force_reinstall, yes=args.yes, ) - if retcode: - return retcode + if process.returncode: + return process.returncode if os.environ.get("CONDA_BUILD_STATE") != "BUILD": ensure_externally_managed(prefix) return 0 diff --git a/conda_pypi/dependencies/pip.py b/conda_pypi/dependencies/pip.py index 8137463..8edca6b 100644 --- a/conda_pypi/dependencies/pip.py +++ b/conda_pypi/dependencies/pip.py @@ -1,15 +1,9 @@ from __future__ import annotations import json -import os from logging import getLogger from collections import defaultdict -from subprocess import run -from tempfile import NamedTemporaryFile - -from conda.exceptions import CondaError - -from ..utils import get_env_python +from ..main import dry_run_pip_json logger = getLogger(f"conda.{__name__}") @@ -19,43 +13,7 @@ def _analyze_with_pip( prefix: str | None = None, force_reinstall: bool = False, ) -> tuple[dict[str, list[str]], dict[str, list[str]]]: - # pip can output to stdout via `--report -` (dash), but this - # creates issues on Windows due to undecodable characters on some - # project descriptions (e.g. charset-normalizer, amusingly), which - # makes pip crash internally. Probably a bug on their end. - # So we use a temporary file instead to work with bytes. - json_output = NamedTemporaryFile(suffix=".json", delete=False) - json_output.close() # Prevent access errors on Windows - - cmd = [ - str(get_env_python(prefix)), - "-mpip", - "install", - "--dry-run", - "--ignore-installed", - *(("--force-reinstall",) if force_reinstall else ()), - "--report", - json_output.name, - *packages, - ] - process = run(cmd, capture_output=True, text=True) - if process.returncode != 0: - raise CondaError( - f"Failed to analyze dependencies with pip:\n" - f" command: {' '.join(cmd)}\n" - f" exit code: {process.returncode}\n" - f" stderr:\n{process.stderr}\n" - f" stdout:\n{process.stdout}\n" - ) - logger.debug("pip (%s) provided the following report:\n%s", " ".join(cmd), process.stdout) - - with open(json_output.name, "rb") as f: - # We need binary mode because the JSON output might - # contain weird unicode stuff (as part of the project - # description or README). - report = json.loads(f.read()) - os.unlink(json_output.name) - + report = dry_run_pip_json(prefix, packages, force_reinstall) deps_from_pip = defaultdict(list) conda_deps = defaultdict(list) for item in report["install"]: diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 327d090..4350134 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -2,10 +2,12 @@ import json import os +import sys from logging import getLogger from pathlib import Path -from subprocess import run -from typing import Iterable +from subprocess import run, CompletedProcess +from tempfile import NamedTemporaryFile, TemporaryDirectory +from typing import Any, Iterable try: from importlib.resources import files as importlib_files @@ -21,7 +23,12 @@ from conda.exceptions import CondaError, CondaSystemExit from conda.models.match_spec import MatchSpec -from .utils import get_env_python, get_externally_managed_path, pypi_spec_variants +from .utils import ( + get_env_python, + get_env_site_packages, + get_externally_managed_path, + pypi_spec_variants, +) logger = getLogger(f"conda.{__name__}") HERE = Path(__file__).parent.resolve() @@ -86,24 +93,28 @@ def run_conda_install( def run_pip_install( prefix: Path, - specs: Iterable[str], + args: Iterable[str], upgrade: bool = False, dry_run: bool = False, quiet: bool = False, verbosity: int = 0, force_reinstall: bool = False, yes: bool = False, -) -> int: - if not specs: + capture_output: bool = False, + check: bool = True +) -> CompletedProcess: + if not args: return 0 command = [ get_env_python(prefix), "-mpip", "install", "--no-deps", - "--prefix", - str(prefix), ] + if any(flag in args for flag in ("--platform", "--abi", "--implementation", "--python-version")): + command += ["--target", str(get_env_site_packages(prefix))] + else: + command += ["--prefix", str(prefix)] if dry_run: command.append("--dry-run") if quiet: @@ -114,11 +125,19 @@ def run_pip_install( command.append("--force-reinstall") if upgrade: command.append("--upgrade") - command.extend(specs) + command.extend(args) logger.info("pip install command: %s", command) - process = run(command) - return process.returncode + process = run(command, capture_output=capture_output or check, text=capture_output or check) + if check and process.returncode: + raise CondaError( + f"Failed to run pip:\n" + f" command: {' '.join(command)}\n" + f" exit code: {process.returncode}\n" + f" stderr:\n{process.stderr}\n" + f" stdout:\n{process.stdout}\n" + ) + return process def ensure_externally_managed(prefix: os.PathLike = None) -> Path: @@ -218,23 +237,64 @@ def pypi_lines_for_explicit_lockfile(prefix: Path | str) -> list[str]: else: seen = {"abi": set(), "platform": set()} lines.append(f"# pypi: {record.name}=={record.version}") - if wheel and (wheel_tag := wheel.get('Tag')): - for i, tag in enumerate(wheel_tag): - python_tag, abi_tag, platform_tag = tag.split("-", 2) - if i == 0: # only once - lines[-1] += f" --python-version {python_tag[2:]}" - lines[-1] += f" --implementation {python_tag[:2]}" - if abi_tag not in seen["abi"]: + lines[-1] += f" --python-version {python_details['version']}" + lines[-1] += f" --implementation {python_details['implementation']}" + if wheel and (wheel_tag := wheel.get("Tag")): + for tag in wheel_tag: + _, abi_tag, platform_tag = tag.split("-", 2) + if abi_tag != "none" and abi_tag not in seen["abi"]: lines[-1] += f" --abi {abi_tag}" seen["abi"].add(abi_tag) - if platform_tag not in seen["platform"]: + if platform_tag != "any" and platform_tag not in seen["platform"]: lines[-1] += f" --platform {platform_tag}" seen["platform"].add(platform_tag) - else: - lines[-1] += f" --python-version {python_details['version']}" - lines[-1] += f" --implementation {python_details['implementation']}" # Here we could try to run a --dry-run --report some.json to get the resolved URL # but it's not guaranteed we get the exact same source so for now we defer to install # time - + return lines + + +def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict[str, Any]: + # pip can output to stdout via `--report -` (dash), but this + # creates issues on Windows due to undecodable characters on some + # project descriptions (e.g. charset-normalizer, amusingly), which + # makes pip crash internally. Probably a bug on their end. + # So we use a temporary file instead to work with bytes. + json_output = NamedTemporaryFile(suffix=".json", delete=False) + json_output.close() # Prevent access errors on Windows + + cmd = [ + sys.executable, + "-mpip", + "install", + "--dry-run", + "--ignore-installed", + *(("--force-reinstall",) if force_reinstall else ()), + "--report", + json_output.name, + *args, + ] + process = run(cmd, capture_output=True, text=True) + if process.returncode != 0: + raise CondaError( + f"Failed to dry-run pip:\n" + f" command: {' '.join(cmd)}\n" + f" exit code: {process.returncode}\n" + f" stderr:\n{process.stderr}\n" + f" stdout:\n{process.stdout}\n" + ) + + with open(json_output.name, "rb") as f: + # We need binary mode because the JSON output might + # contain weird unicode stuff (as part of the project + # description or README). + report = json.loads(f.read()) + os.unlink(json_output.name) + return report + + +def pip_install_download_info(args: Iterable[str]) -> dict[str, Any]: + with TemporaryDirectory() as tmpdir: + report = dry_run_pip_json(["--target", tmpdir, "--no-deps", *args]) + return report["install"][0]["download_info"] diff --git a/conda_pypi/utils.py b/conda_pypi/utils.py index 327bda3..ed025aa 100644 --- a/conda_pypi/utils.py +++ b/conda_pypi/utils.py @@ -31,22 +31,30 @@ def get_env_python(prefix: os.PathLike = None) -> Path: return prefix / "bin" / "python" -def get_env_stdlib(prefix: os.PathLike = None) -> Path: +def _get_env_sysconfig_path(key: str, prefix: os.PathLike = None) -> Path: prefix = Path(prefix or sys.prefix) if str(prefix) == sys.prefix: - return Path(sysconfig.get_path("stdlib")) + return Path(sysconfig.get_path(key)) return Path( check_output( [ get_env_python(prefix), "-c", - "import sysconfig; print(sysconfig.get_paths()['stdlib'])", + f"import sysconfig; print(sysconfig.get_paths()['{key}'])", ], text=True, ).strip() ) +def get_env_stdlib(prefix: os.PathLike = None) -> Path: + return _get_env_sysconfig_path("stdlib", prefix) + + +def get_env_site_packages(prefix: os.PathLike = None) -> Path: + return _get_env_sysconfig_path("purelib", prefix) + + def get_externally_managed_path(prefix: os.PathLike = None) -> Iterator[Path]: prefix = Path(prefix or sys.prefix) if os.name == "nt": From 687592734eaca00aa84bd00b41c07bf414b6e5a1 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 18 May 2024 10:07:46 +0200 Subject: [PATCH 04/22] fix call --- conda_pypi/dependencies/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_pypi/dependencies/pip.py b/conda_pypi/dependencies/pip.py index 8edca6b..fbd1c9b 100644 --- a/conda_pypi/dependencies/pip.py +++ b/conda_pypi/dependencies/pip.py @@ -13,7 +13,7 @@ def _analyze_with_pip( prefix: str | None = None, force_reinstall: bool = False, ) -> tuple[dict[str, list[str]], dict[str, list[str]]]: - report = dry_run_pip_json(prefix, packages, force_reinstall) + report = dry_run_pip_json(("--prefix", prefix, *packages), force_reinstall) deps_from_pip = defaultdict(list) conda_deps = defaultdict(list) for item in report["install"]: From 6364f7fca45dfc07c02ed1cf7c2705af7c55279f Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 18 May 2024 11:04:47 +0200 Subject: [PATCH 05/22] add test --- tests/test_install.py | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/tests/test_install.py b/tests/test_install.py index 5f96093..a72cf44 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,14 +1,17 @@ from __future__ import annotations import sys +from pathlib import Path +from subprocess import run +from typing import Iterable import pytest - from conda.core.prefix_data import PrefixData from conda.models.match_spec import MatchSpec from conda.testing import CondaCLIFixture, TmpEnvFixture from conda_pypi.dependencies import NAME_MAPPINGS, BACKENDS, _pypi_spec_to_conda_spec +from conda_pypi.utils import get_env_python @pytest.mark.parametrize("source", NAME_MAPPINGS.keys()) @@ -123,3 +126,38 @@ def test_pyqt( for conda_spec in installed_conda_specs: assert conda_spec in out + +@pytest.mark.parametrize( + "specs, pure_pip", + [ + (("requests",), True), + (("requests",), False), + ] +) +def test_lockfile_roundtrip( + tmp_path: Path, + tmp_env: TmpEnvFixture, + conda_cli: CondaCLIFixture, + specs: Iterable[str], + pure_pip: bool, +): + with tmp_env("python=3.9", "pip") as prefix: + if pure_pip: + p = run([get_env_python(prefix), "-mpip", "install", "--break-system-packages", *specs]) + p.check_returncode() + else: + out, err, rc = conda_cli("pip", "--prefix", prefix, "--yes", "install", *specs) + print(out) + print(err, file=sys.stderr) + assert rc == 0 + out, err, rc = conda_cli("list", "--explicit") + print(out) + print(err, file=sys.stderr) + assert rc == 0 + (tmp_path / "lockfile.txt").write_text(out) + p = run([sys.executable, "-mconda", "create", "--prefix", tmp_path / "env", "--file", tmp_path / "lockfile.txt"]) + out2, err2, rc2 = conda_cli("list", "--explicit", "--prefix", tmp_path / "env") + print(out2) + print(err2, file=sys.stderr) + assert rc2 == 0 + assert out2 == out From 563238a60bc05082f21983af11d9e8feae1fa8de Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 18 May 2024 13:36:23 +0200 Subject: [PATCH 06/22] order can vary across runs :shrug: --- tests/test_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_install.py b/tests/test_install.py index a72cf44..bc37dd3 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -160,4 +160,4 @@ def test_lockfile_roundtrip( print(out2) print(err2, file=sys.stderr) assert rc2 == 0 - assert out2 == out + assert sorted(out2.splitlines()) == sorted(out.splitlines()) From faa1899037ee1c9c683697e55708217693ae2392 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 22 May 2024 17:16:47 +0200 Subject: [PATCH 07/22] add disclaimer --- conda_pypi/cli/install.py | 6 ++++-- conda_pypi/cli/list.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index b125cca..a3dd03f 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -14,12 +14,14 @@ from typing import Iterable -def post_command(command: str): +def post_command(command: str) -> int: if command not in ("install", "create"): - return + return 0 installed = [] pypi_lines = pypi_lines_from_sys_argv() + if not pypi_lines: + return 0 with Spinner( f"Installing PyPI packages ({len(pypi_lines)})", diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index e916aeb..252b7a7 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -13,4 +13,5 @@ def post_command(command: str): if to_print: print("# Following lines added by conda-pypi") + print("# This is an experimental feature subject to change. Do not use in production.") print(*to_print, sep="\n") From 608f27ee24a87d595d9349372df954e26bfba4ea Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 22 May 2024 17:21:39 +0200 Subject: [PATCH 08/22] add version to disclaimer --- conda_pypi/__init__.py | 1 + conda_pypi/cli/list.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda_pypi/__init__.py b/conda_pypi/__init__.py index b6d2aa1..fbb80c0 100644 --- a/conda_pypi/__init__.py +++ b/conda_pypi/__init__.py @@ -1,3 +1,4 @@ """ conda-pypi """ +__version__ = "0.0.0" \ No newline at end of file diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index 252b7a7..ff21ba0 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -2,6 +2,7 @@ from conda.base.context import context +from .. import __version__ from ..main import pypi_lines_for_explicit_lockfile def post_command(command: str): @@ -12,6 +13,6 @@ def post_command(command: str): to_print = pypi_lines_for_explicit_lockfile(context.target_prefix) if to_print: - print("# Following lines added by conda-pypi") + print(f"# The following lines were added by conda-pypi v{__version__}") print("# This is an experimental feature subject to change. Do not use in production.") print(*to_print, sep="\n") From abb77edac3e0098bf37dd2ec1c8b39dcd2ce2a3c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 22 May 2024 17:41:04 +0200 Subject: [PATCH 09/22] add md5 of the RECORD file --- conda_pypi/cli/list.py | 6 +++++- conda_pypi/main.py | 14 ++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index ff21ba0..9aaca04 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -5,12 +5,16 @@ from .. import __version__ from ..main import pypi_lines_for_explicit_lockfile + def post_command(command: str): if command != "list": return if "--explicit" not in sys.argv: return - to_print = pypi_lines_for_explicit_lockfile(context.target_prefix) + if "--no-pip" in sys.argv: + return + checksum = "md5" if "--md5" in sys.argv else None + to_print = pypi_lines_for_explicit_lockfile(context.target_prefix, checksum=checksum) if to_print: print(f"# The following lines were added by conda-pypi v{__version__}") diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 4350134..914ccf3 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -7,7 +7,7 @@ from pathlib import Path from subprocess import run, CompletedProcess from tempfile import NamedTemporaryFile, TemporaryDirectory -from typing import Any, Iterable +from typing import Any, Iterable, Literal try: from importlib.resources import files as importlib_files @@ -17,6 +17,7 @@ from conda.base.context import context from conda.core.prefix_data import PrefixData +from conda.gateways.disk.read import compute_sum from conda.models.enums import PackageType from conda.history import History from conda.cli.python_api import run_command @@ -192,7 +193,7 @@ def ensure_target_env_has_externally_managed(command: str): raise ValueError(f"command {command} not recognized.") -def pypi_lines_for_explicit_lockfile(prefix: Path | str) -> list[str]: +def pypi_lines_for_explicit_lockfile(prefix: Path | str, checksum: Literal["md5", "sha256"] | None = None) -> list[str]: PrefixData._cache_.clear() pd = PrefixData(str(prefix), pip_interop_enabled=True) pd.load() @@ -210,17 +211,20 @@ def pypi_lines_for_explicit_lockfile(prefix: Path | str) -> list[str]: continue ignore = False wheel = {} + hashed_record = "" for path in record.files: path = Path(context.target_prefix, path) if "__editable__" in path.stem: ignore = True break - if path.name == "direct_url.json": + if path.name == "direct_url.json" and path.parent.suffix == ".dist-info": data = json.loads(path.read_text()) if data.get("dir_info", {}).get("editable"): ignore = True break - if path.name == "WHEEL": + if checksum and path.name == "RECORD" and path.parent.suffix == ".dist-info": + hashed_record = compute_sum(path, checksum) + if path.name == "WHEEL" and path.parent.suffix == ".dist-info": for line in path.read_text().splitlines(): line = line.strip() if ":" not in line: @@ -251,6 +255,8 @@ def pypi_lines_for_explicit_lockfile(prefix: Path | str) -> list[str]: # Here we could try to run a --dry-run --report some.json to get the resolved URL # but it's not guaranteed we get the exact same source so for now we defer to install # time + if checksum and hashed_record: + lines[-1] += f" -- --checksum={checksum}:{hashed_record}" return lines From c15ef183d3a9d3016f6cad8d37e08122f7aa6725 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 22 May 2024 17:42:52 +0200 Subject: [PATCH 10/22] test with --md5 --- tests/test_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_install.py b/tests/test_install.py index bc37dd3..1783673 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -150,7 +150,7 @@ def test_lockfile_roundtrip( print(out) print(err, file=sys.stderr) assert rc == 0 - out, err, rc = conda_cli("list", "--explicit") + out, err, rc = conda_cli("list", "--explicit", "--md5") print(out) print(err, file=sys.stderr) assert rc == 0 From d3de85e310821a7bba658132753a20b04e13f420 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 May 2024 15:06:06 +0200 Subject: [PATCH 11/22] pre-commit --- conda_pypi/__init__.py | 3 ++- conda_pypi/cli/__init__.py | 2 ++ conda_pypi/cli/install.py | 3 +++ conda_pypi/main.py | 12 ++++++++---- tests/test_install.py | 30 ++++++++++++++++++++++-------- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/conda_pypi/__init__.py b/conda_pypi/__init__.py index fbb80c0..ee90b70 100644 --- a/conda_pypi/__init__.py +++ b/conda_pypi/__init__.py @@ -1,4 +1,5 @@ """ conda-pypi """ -__version__ = "0.0.0" \ No newline at end of file + +__version__ = "0.0.0" diff --git a/conda_pypi/cli/__init__.py b/conda_pypi/cli/__init__.py index 3767cfa..cb72ca6 100644 --- a/conda_pypi/cli/__init__.py +++ b/conda_pypi/cli/__init__.py @@ -1 +1,3 @@ from . import install, list, pip + +__all__ = ["install", "list", "pip"] diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index a3dd03f..3589fd5 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -30,6 +30,9 @@ def post_command(command: str) -> int: ): for args in pypi_lines: args = shlex.split(args) + double_dash_position = args.find("--") + if double_dash_position >= 0: + args = args[:double_dash_position] dl_info = pip_install_download_info(args) run_pip_install( context.target_prefix, diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 914ccf3..79d42d2 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -102,7 +102,7 @@ def run_pip_install( force_reinstall: bool = False, yes: bool = False, capture_output: bool = False, - check: bool = True + check: bool = True, ) -> CompletedProcess: if not args: return 0 @@ -112,7 +112,9 @@ def run_pip_install( "install", "--no-deps", ] - if any(flag in args for flag in ("--platform", "--abi", "--implementation", "--python-version")): + if any( + flag in args for flag in ("--platform", "--abi", "--implementation", "--python-version") + ): command += ["--target", str(get_env_site_packages(prefix))] else: command += ["--prefix", str(prefix)] @@ -193,7 +195,9 @@ def ensure_target_env_has_externally_managed(command: str): raise ValueError(f"command {command} not recognized.") -def pypi_lines_for_explicit_lockfile(prefix: Path | str, checksum: Literal["md5", "sha256"] | None = None) -> list[str]: +def pypi_lines_for_explicit_lockfile( + prefix: Path | str, checksum: Literal["md5", "sha256"] | None = None +) -> list[str]: PrefixData._cache_.clear() pd = PrefixData(str(prefix), pip_interop_enabled=True) pd.load() @@ -211,7 +215,7 @@ def pypi_lines_for_explicit_lockfile(prefix: Path | str, checksum: Literal["md5" continue ignore = False wheel = {} - hashed_record = "" + hashed_record = "" for path in record.files: path = Path(context.target_prefix, path) if "__editable__" in path.stem: diff --git a/tests/test_install.py b/tests/test_install.py index 1783673..7434ad7 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -19,14 +19,15 @@ def test_mappings_one_by_one(source: str): assert _pypi_spec_to_conda_spec("build", sources=(source,)) == "python-build" -@pytest.mark.parametrize("pypi_spec,conda_spec", +@pytest.mark.parametrize( + "pypi_spec,conda_spec", [ ("numpy", "numpy"), ("build", "python-build"), ("ib_insync", "ib-insync"), ("pyqt5", "pyqt>=5.0.0,<6.0.0.0dev0"), ("PyQt5", "pyqt>=5.0.0,<6.0.0.0dev0"), - ] + ], ) def test_mappings_fallback(pypi_spec: str, conda_spec: str): assert MatchSpec(_pypi_spec_to_conda_spec(pypi_spec)) == MatchSpec(conda_spec) @@ -105,10 +106,11 @@ def test_spec_normalization( assert "All packages are already installed." in out + err -@pytest.mark.parametrize("pypi_spec,requested_conda_spec,installed_conda_specs", +@pytest.mark.parametrize( + "pypi_spec,requested_conda_spec,installed_conda_specs", [ ("PyQt5", "pyqt[version='>=5.0.0,<6.0.0.0dev0']", ("pyqt-5", "qt-main-5")), - ] + ], ) def test_pyqt( tmp_env: TmpEnvFixture, @@ -125,14 +127,14 @@ def test_pyqt( assert requested_conda_spec in out for conda_spec in installed_conda_specs: assert conda_spec in out - + @pytest.mark.parametrize( "specs, pure_pip", [ (("requests",), True), (("requests",), False), - ] + ], ) def test_lockfile_roundtrip( tmp_path: Path, @@ -143,7 +145,9 @@ def test_lockfile_roundtrip( ): with tmp_env("python=3.9", "pip") as prefix: if pure_pip: - p = run([get_env_python(prefix), "-mpip", "install", "--break-system-packages", *specs]) + p = run( + [get_env_python(prefix), "-mpip", "install", "--break-system-packages", *specs] + ) p.check_returncode() else: out, err, rc = conda_cli("pip", "--prefix", prefix, "--yes", "install", *specs) @@ -155,7 +159,17 @@ def test_lockfile_roundtrip( print(err, file=sys.stderr) assert rc == 0 (tmp_path / "lockfile.txt").write_text(out) - p = run([sys.executable, "-mconda", "create", "--prefix", tmp_path / "env", "--file", tmp_path / "lockfile.txt"]) + p = run( + [ + sys.executable, + "-mconda", + "create", + "--prefix", + tmp_path / "env", + "--file", + tmp_path / "lockfile.txt", + ] + ) out2, err2, rc2 = conda_cli("list", "--explicit", "--prefix", tmp_path / "env") print(out2) print(err2, file=sys.stderr) From 9534226a21ce1247d4216ebeba307ed336b6e05d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 May 2024 16:30:40 +0200 Subject: [PATCH 12/22] Verify checksums of RECORD files upon install --- conda_pypi/cli/install.py | 95 ++++++++++++++++++++++++++++----------- conda_pypi/main.py | 37 ++++++++++----- 2 files changed, 97 insertions(+), 35 deletions(-) diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index 3589fd5..1518947 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -2,50 +2,95 @@ import shlex import sys +from logging import getLogger from pathlib import Path from typing import TYPE_CHECKING from conda.base.context import context from conda.common.io import Spinner -from ..main import run_pip_install, pip_install_download_info +from ..main import run_pip_install, dry_run_pip_json, compute_record_sum +from ..utils import get_env_site_packages if TYPE_CHECKING: from typing import Iterable +log = getLogger(f"conda.{__name__}") + + +def _prepare_pypi_transaction(pypi_lines) -> dict[str, dict[str, str]]: + pkgs = {} + for args in pypi_lines: + args = shlex.split(args) + record_hash = None + if "--" in args: + double_dash_idx = args.index("--") + if double_dash_idx >= 0: + args, extra_args = args[:double_dash_idx], args[double_dash_idx:] + if ( + "--checksum" in extra_args + and (hash_idx := extra_args.index("--checksum")) > 0 + and extra_args[hash_idx + 1].startswith(("md5:", "sha256:")) + ): + record_hash = extra_args[hash_idx + 1] + else: + for arg in extra_args: + if arg.startswith("--checksum="): + record_hash = arg.split("=", 1)[1] + report = dry_run_pip_json(["--no-deps", *args]) + pkg_name = report["install"][0]["metadata"]["name"] + version = report["install"][0]["metadata"]["version"] + pkgs[(pkg_name, version)] = {"url": report["install"][0]["download_info"]["url"]} + if record_hash: + pkgs[(pkg_name, version)]["hash"] = record_hash + return pkgs + def post_command(command: str) -> int: if command not in ("install", "create"): return 0 - installed = [] pypi_lines = pypi_lines_from_sys_argv() if not pypi_lines: return 0 - with Spinner( - f"Installing PyPI packages ({len(pypi_lines)})", - enabled=not context.quiet, - json=context.json, - ): - for args in pypi_lines: - args = shlex.split(args) - double_dash_position = args.find("--") - if double_dash_position >= 0: - args = args[:double_dash_position] - dl_info = pip_install_download_info(args) - run_pip_install( - context.target_prefix, - args=[dl_info["url"]], - dry_run=context.dry_run, - quiet=context.quiet, - verbosity=context.verbosity, - force_reinstall=context.force_reinstall, - yes=context.always_yes, - check=True, - ) - installed.append(args[0]) - print("Successfully installed PyPI packages:", *installed) + with Spinner("\nPreparing PyPI transaction", enabled=not context.quiet, json=context.json): + pkgs = _prepare_pypi_transaction(pypi_lines) + + with Spinner("Executing PyPI transaction", enabled=not context.quiet, json=context.json): + run_pip_install( + context.target_prefix, + args=[pkg["url"] for pkg in pkgs.values()], + dry_run=context.dry_run, + quiet=context.quiet, + verbosity=context.verbosity, + force_reinstall=context.force_reinstall, + yes=context.always_yes, + check=True, + ) + + if any(pkg.get("hash") for pkg in pkgs.values()): + with Spinner("Verifying PyPI transaction", enabled=not context.quiet, json=context.json): + site_packages = get_env_site_packages(context.target_prefix) + for dist_info in site_packages.glob("*.dist-info"): + if not dist_info.is_dir(): + continue + name, version = dist_info.stem.split("-") + expected_hash = pkgs.get((name, version), {}).get("hash") + if expected_hash: + algo, expected_hash = expected_hash.split(":") + if (dist_info / "RECORD").is_file(): + found_hash = compute_record_sum(dist_info / "RECORD", algo) + if expected_hash != found_hash: + log.warning( + "%s checksum for %s==%s didn't match! Expected=%s, found=%s", + algo, + name, + version, + expected_hash, + found_hash, + ) + return 0 diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 79d42d2..5cd3cdd 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -2,11 +2,12 @@ import json import os +import shlex import sys from logging import getLogger from pathlib import Path from subprocess import run, CompletedProcess -from tempfile import NamedTemporaryFile, TemporaryDirectory +from tempfile import NamedTemporaryFile from typing import Any, Iterable, Literal try: @@ -135,10 +136,10 @@ def run_pip_install( if check and process.returncode: raise CondaError( f"Failed to run pip:\n" - f" command: {' '.join(command)}\n" + f" command: {shlex.join(command)}\n" f" exit code: {process.returncode}\n" f" stderr:\n{process.stderr}\n" - f" stdout:\n{process.stdout}\n" + f" stdout:\n{process.stdout}" ) return process @@ -227,7 +228,7 @@ def pypi_lines_for_explicit_lockfile( ignore = True break if checksum and path.name == "RECORD" and path.parent.suffix == ".dist-info": - hashed_record = compute_sum(path, checksum) + hashed_record = compute_record_sum(path, checksum) if path.name == "WHEEL" and path.parent.suffix == ".dist-info": for line in path.read_text().splitlines(): line = line.strip() @@ -283,16 +284,18 @@ def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict *(("--force-reinstall",) if force_reinstall else ()), "--report", json_output.name, + "--target", + json_output.name + ".dir", *args, ] process = run(cmd, capture_output=True, text=True) if process.returncode != 0: raise CondaError( f"Failed to dry-run pip:\n" - f" command: {' '.join(cmd)}\n" + f" command: {shlex.join(cmd)}\n" f" exit code: {process.returncode}\n" f" stderr:\n{process.stderr}\n" - f" stdout:\n{process.stdout}\n" + f" stdout:\n{process.stdout}" ) with open(json_output.name, "rb") as f: @@ -304,7 +307,21 @@ def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict return report -def pip_install_download_info(args: Iterable[str]) -> dict[str, Any]: - with TemporaryDirectory() as tmpdir: - report = dry_run_pip_json(["--target", tmpdir, "--no-deps", *args]) - return report["install"][0]["download_info"] +def compute_record_sum(record_path, algo): + record = Path(record_path).read_text() + lines = [] + for line in record.splitlines(): + path, *_ = line.split(",") + path = Path(path) + if path.parts[0].endswith(".dist-info") and path.name in ("REQUESTED", "direct_url.json"): + continue + if path.parts[0] == ".." and "bin" in path.parts: + continue + lines.append(line) + with NamedTemporaryFile("w", delete=False) as tmp: + tmp.write("\n".join(lines)) + + try: + return compute_sum(tmp.name, algo) + finally: + os.unlink(tmp.name) From 545deb22a1ccf56eadd468438616fd835a3e1bee Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 23 May 2024 17:18:03 +0200 Subject: [PATCH 13/22] fix test --- tests/test_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_install.py b/tests/test_install.py index 7434ad7..7b892f7 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -170,7 +170,7 @@ def test_lockfile_roundtrip( tmp_path / "lockfile.txt", ] ) - out2, err2, rc2 = conda_cli("list", "--explicit", "--prefix", tmp_path / "env") + out2, err2, rc2 = conda_cli("list", "--explicit", "--md5", "--prefix", tmp_path / "env") print(out2) print(err2, file=sys.stderr) assert rc2 == 0 From 8b1cb98b1e44f3655c379db1014412e218a84266 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 24 May 2024 10:11:39 +0200 Subject: [PATCH 14/22] fix test --- conda_pypi/cli/install.py | 6 +++--- conda_pypi/cli/list.py | 10 +++++----- conda_pypi/main.py | 12 ++++++++---- tests/test_install.py | 41 ++++++++++++++++++++++++++++++--------- 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index 1518947..914b8d3 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -28,14 +28,14 @@ def _prepare_pypi_transaction(pypi_lines) -> dict[str, dict[str, str]]: if double_dash_idx >= 0: args, extra_args = args[:double_dash_idx], args[double_dash_idx:] if ( - "--checksum" in extra_args - and (hash_idx := extra_args.index("--checksum")) > 0 + "--record-checksum" in extra_args + and (hash_idx := extra_args.index("--record-checksum")) > 0 and extra_args[hash_idx + 1].startswith(("md5:", "sha256:")) ): record_hash = extra_args[hash_idx + 1] else: for arg in extra_args: - if arg.startswith("--checksum="): + if arg.startswith("--record-checksum="): record_hash = arg.split("=", 1)[1] report = dry_run_pip_json(["--no-deps", *args]) pkg_name = report["install"][0]["metadata"]["name"] diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index 9aaca04..ecd5443 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -1,5 +1,4 @@ import sys - from conda.base.context import context from .. import __version__ @@ -9,14 +8,15 @@ def post_command(command: str): if command != "list": return - if "--explicit" not in sys.argv: + cmd_line = context.raw_data.get("cmd_line", {}) + if "--explicit" not in sys.argv and not cmd_line.get("explicit"): return - if "--no-pip" in sys.argv: + if "--no-pip" in sys.argv or not cmd_line.get("pip"): return - checksum = "md5" if "--md5" in sys.argv else None + checksum = "md5" if ("--md5" in sys.argv or cmd_line.get("md5")) else None to_print = pypi_lines_for_explicit_lockfile(context.target_prefix, checksum=checksum) - if to_print: + sys.stdout.flush() print(f"# The following lines were added by conda-pypi v{__version__}") print("# This is an experimental feature subject to change. Do not use in production.") print(*to_print, sep="\n") diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 5cd3cdd..1b6ee37 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -261,7 +261,7 @@ def pypi_lines_for_explicit_lockfile( # but it's not guaranteed we get the exact same source so for now we defer to install # time if checksum and hashed_record: - lines[-1] += f" -- --checksum={checksum}:{hashed_record}" + lines[-1] += f" -- --record-checksum={checksum}:{hashed_record}" return lines @@ -313,10 +313,14 @@ def compute_record_sum(record_path, algo): for line in record.splitlines(): path, *_ = line.split(",") path = Path(path) - if path.parts[0].endswith(".dist-info") and path.name in ("REQUESTED", "direct_url.json"): - continue - if path.parts[0] == ".." and "bin" in path.parts: + if path.parts[0].endswith(".dist-info") and path.name not in ("METADATA", "WHEEL"): + # we only want to check the metadata and wheel parts of dist-info; everything else + # is not deterministic or useful continue + if path.parts[0] == ".." and ("bin" in path.parts or "lib" in path.parts): + # entry points are autogenerated and can have different hashes/size depending on prefix + path, *_ = line.split(",") + line = f"{path},," lines.append(line) with NamedTemporaryFile("w", delete=False) as tmp: tmp.write("\n".join(lines)) diff --git a/tests/test_install.py b/tests/test_install.py index 7b892f7..32f56f9 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -130,10 +130,12 @@ def test_pyqt( @pytest.mark.parametrize( - "specs, pure_pip", + "specs, pure_pip, with_md5", [ - (("requests",), True), - (("requests",), False), + (("requests",), True, True), + (("requests",), False, False), + (("requests",), True, False), + (("requests",), False, True), ], ) def test_lockfile_roundtrip( @@ -142,23 +144,35 @@ def test_lockfile_roundtrip( conda_cli: CondaCLIFixture, specs: Iterable[str], pure_pip: bool, + with_md5: bool, ): + md5 = ("--md5",) if with_md5 else () with tmp_env("python=3.9", "pip") as prefix: if pure_pip: p = run( - [get_env_python(prefix), "-mpip", "install", "--break-system-packages", *specs] + [get_env_python(prefix), "-mpip", "install", "--break-system-packages", *specs], + capture_output=True, + text=True, + check=False, ) - p.check_returncode() + print(p.stdout) + print(p.stderr, file=sys.stderr) + assert p.returncode == 0 else: out, err, rc = conda_cli("pip", "--prefix", prefix, "--yes", "install", *specs) print(out) print(err, file=sys.stderr) assert rc == 0 - out, err, rc = conda_cli("list", "--explicit", "--md5") + out, err, rc = conda_cli("list", "--explicit", "--prefix", prefix, *md5) print(out) print(err, file=sys.stderr) assert rc == 0 - (tmp_path / "lockfile.txt").write_text(out) + if pure_pip: + assert "# pypi: requests" in out + if md5: + assert "--record-checksum=md5:" in out + + (tmp_path / "lockfile.txt").write_text(out) p = run( [ sys.executable, @@ -168,9 +182,18 @@ def test_lockfile_roundtrip( tmp_path / "env", "--file", tmp_path / "lockfile.txt", - ] + ], + capture_output=True, + text=True, + check=False, ) - out2, err2, rc2 = conda_cli("list", "--explicit", "--md5", "--prefix", tmp_path / "env") + print(p.stdout) + print(p.stderr, file=sys.stderr) + assert p.returncode == 0 + if pure_pip: + assert "Verifying PyPI transaction" in p.stdout + + out2, err2, rc2 = conda_cli("list", "--explicit", *md5, "--prefix", tmp_path / "env") print(out2) print(err2, file=sys.stderr) assert rc2 == 0 From 9fd169ce4c2030321d1a50950bdd29caa38930ca Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 27 May 2024 12:08:05 +0200 Subject: [PATCH 15/22] fix windows entry points --- conda_pypi/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 1b6ee37..3818d54 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -317,7 +317,7 @@ def compute_record_sum(record_path, algo): # we only want to check the metadata and wheel parts of dist-info; everything else # is not deterministic or useful continue - if path.parts[0] == ".." and ("bin" in path.parts or "lib" in path.parts): + if path.parts[0] == ".." and any(part in path.parts for part in ("bin", "lib", "Scripts")): # entry points are autogenerated and can have different hashes/size depending on prefix path, *_ = line.split(",") line = f"{path},," From 70c24d606f484ffb8600d25ee053d23e37e532b2 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 29 May 2024 11:58:14 +0200 Subject: [PATCH 16/22] fix test --- tests/test_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_install.py b/tests/test_install.py index 32f56f9..9e4c016 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -190,7 +190,7 @@ def test_lockfile_roundtrip( print(p.stdout) print(p.stderr, file=sys.stderr) assert p.returncode == 0 - if pure_pip: + if pure_pip and with_md5: assert "Verifying PyPI transaction" in p.stdout out2, err2, rc2 = conda_cli("list", "--explicit", *md5, "--prefix", tmp_path / "env") From 43451ab5288ab4542f6f186c365f037cf6fa20ef Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 29 May 2024 11:58:38 +0200 Subject: [PATCH 17/22] fix `context` cmd_line value fetching --- conda_pypi/cli/list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index ecd5443..ec9fb44 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -9,11 +9,11 @@ def post_command(command: str): if command != "list": return cmd_line = context.raw_data.get("cmd_line", {}) - if "--explicit" not in sys.argv and not cmd_line.get("explicit"): + if "--explicit" not in sys.argv and not cmd_line.get("explicit").value(None): return if "--no-pip" in sys.argv or not cmd_line.get("pip"): return - checksum = "md5" if ("--md5" in sys.argv or cmd_line.get("md5")) else None + checksum = "md5" if ("--md5" in sys.argv or cmd_line.get("md5").value(None)) else None to_print = pypi_lines_for_explicit_lockfile(context.target_prefix, checksum=checksum) if to_print: sys.stdout.flush() From 69e16618017f743f847c1cf08362fde300185860 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 29 May 2024 11:58:48 +0200 Subject: [PATCH 18/22] use 0.1.0 as t0 --- conda_pypi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_pypi/__init__.py b/conda_pypi/__init__.py index ee90b70..b2d8e7f 100644 --- a/conda_pypi/__init__.py +++ b/conda_pypi/__init__.py @@ -2,4 +2,4 @@ conda-pypi """ -__version__ = "0.0.0" +__version__ = "0.1.0" From 0c49317f33f0ecd34ef066e7df358e816d514414 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 29 May 2024 11:59:31 +0200 Subject: [PATCH 19/22] refactor pypi_lines with a helper object and rely on 3rd party parsers when feasible --- conda_pypi/main.py | 311 +++++++++++++++++++++++++++++++-------------- 1 file changed, 214 insertions(+), 97 deletions(-) diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 3818d54..092985f 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -4,6 +4,8 @@ import os import shlex import sys +from csv import reader as csv_reader +from email.parser import HeaderParser from logging import getLogger from pathlib import Path from subprocess import run, CompletedProcess @@ -17,13 +19,16 @@ from conda.base.context import context +from conda.common.pkg_formats.python import PythonDistribution from conda.core.prefix_data import PrefixData from conda.gateways.disk.read import compute_sum from conda.models.enums import PackageType +from conda.models.records import PackageRecord from conda.history import History from conda.cli.python_api import run_command from conda.exceptions import CondaError, CondaSystemExit from conda.models.match_spec import MatchSpec +from packaging.tags import parse_tag from .utils import ( get_env_python, @@ -206,63 +211,15 @@ def pypi_lines_for_explicit_lockfile( python_record = list(pd.query("python")) assert len(python_record) == 1 python_record = python_record[0] - python_details = {"version": ".".join(python_record.version.split(".")[:3])} - if "pypy" in python_record.build: - python_details["implementation"] = "pp" - else: - python_details["implementation"] = "cp" for record in pd.iter_records(): if record.package_type != PackageType.VIRTUAL_PYTHON_WHEEL: continue - ignore = False - wheel = {} - hashed_record = "" - for path in record.files: - path = Path(context.target_prefix, path) - if "__editable__" in path.stem: - ignore = True - break - if path.name == "direct_url.json" and path.parent.suffix == ".dist-info": - data = json.loads(path.read_text()) - if data.get("dir_info", {}).get("editable"): - ignore = True - break - if checksum and path.name == "RECORD" and path.parent.suffix == ".dist-info": - hashed_record = compute_record_sum(path, checksum) - if path.name == "WHEEL" and path.parent.suffix == ".dist-info": - for line in path.read_text().splitlines(): - line = line.strip() - if ":" not in line: - continue - key, value = line.split(":", 1) - if key == "Tag": - wheel.setdefault(key, []).append(value.strip()) - else: - wheel[key] = value.strip() - if ignore: + pypi_dist = PyPIDistribution.from_conda_record( + record, python_record, prefix, checksum=checksum + ) + if pypi_dist.editable: continue - if record.url: - lines.append(f"# pypi: {record.url}") - else: - seen = {"abi": set(), "platform": set()} - lines.append(f"# pypi: {record.name}=={record.version}") - lines[-1] += f" --python-version {python_details['version']}" - lines[-1] += f" --implementation {python_details['implementation']}" - if wheel and (wheel_tag := wheel.get("Tag")): - for tag in wheel_tag: - _, abi_tag, platform_tag = tag.split("-", 2) - if abi_tag != "none" and abi_tag not in seen["abi"]: - lines[-1] += f" --abi {abi_tag}" - seen["abi"].add(abi_tag) - if platform_tag != "any" and platform_tag not in seen["platform"]: - lines[-1] += f" --platform {platform_tag}" - seen["platform"].add(platform_tag) - # Here we could try to run a --dry-run --report some.json to get the resolved URL - # but it's not guaranteed we get the exact same source so for now we defer to install - # time - if checksum and hashed_record: - lines[-1] += f" -- --record-checksum={checksum}:{hashed_record}" - + lines.append(pypi_dist.to_lockfile_line()) return lines @@ -275,57 +232,217 @@ def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict json_output = NamedTemporaryFile(suffix=".json", delete=False) json_output.close() # Prevent access errors on Windows - cmd = [ - sys.executable, - "-mpip", - "install", - "--dry-run", - "--ignore-installed", - *(("--force-reinstall",) if force_reinstall else ()), - "--report", - json_output.name, - "--target", - json_output.name + ".dir", - *args, - ] - process = run(cmd, capture_output=True, text=True) - if process.returncode != 0: - raise CondaError( - f"Failed to dry-run pip:\n" - f" command: {shlex.join(cmd)}\n" - f" exit code: {process.returncode}\n" - f" stderr:\n{process.stderr}\n" - f" stdout:\n{process.stdout}" + try: + cmd = [ + sys.executable, + "-mpip", + "install", + "--dry-run", + "--ignore-installed", + *(("--force-reinstall",) if force_reinstall else ()), + "--report", + json_output.name, + "--target", + json_output.name + ".dir", # This won't be created + *args, + ] + process = run(cmd, capture_output=True, text=True) + if process.returncode != 0: + raise CondaError( + f"Failed to dry-run pip:\n" + f" command: {shlex.join(cmd)}\n" + f" exit code: {process.returncode}\n" + f" stderr:\n{process.stderr}\n" + f" stdout:\n{process.stdout}" + ) + + with open(json_output.name, "rb") as f: + # We need binary mode because the JSON output might + # contain weird unicode stuff (as part of the project + # description or README). + return json.loads(f.read()) + finally: + os.unlink(json_output.name) + + +class PyPIDistribution: + _line_prefix = "# pypi: " + + def __init__( + self, + name: str, + version: str, + python_version: str | None = None, + python_implementation: str | None = None, + python_abi_tags: Iterable[str] = (), + python_platform_tags: Iterable[str] = (), + files_hash: str | None = None, + editable: bool = False, + ): + self.name = name + self.version = version + self.python_version = python_version + self.python_implementation = python_implementation + self.python_abi_tags = python_abi_tags or () + self.python_platform_tags = python_platform_tags or () + self.files_hash = files_hash + self.editable = editable + self.url = None # currently no way to know + + @classmethod + def from_conda_record( + cls, + record: PackageRecord, + python_record: PackageRecord, + prefix: str | Path, + checksum: Literal["md5", "sha256"] | None = None, + ) -> PyPIDistribution: + # Locate anchor file + sitepackages = get_env_site_packages(prefix) + if record.fn.endswith(".dist-info"): + anchor = sitepackages / record.fn / "METADATA" + elif record.fn.endswith(".egg-info"): + anchor = sitepackages / record.fn + if anchor.is_dir(): + anchor = anchor / "PKG-INFO" + else: + raise ValueError("Unrecognized anchor file for Python metadata") + + # Estimate python implementation out of build strings + python_version = ".".join(python_record.version.split(".")[:3]) + if "pypy" in python_record.build: + python_impl = "pp" + elif "cpython" in python_record.build: + python_impl = "cp" + else: + python_impl = None + + # Find the hash for the RECORD file + python_dist = PythonDistribution.init(prefix, str(anchor), python_record.version) + if checksum: + manifest = python_dist.manifest_full_path + hashed_files = f"{checksum}:{compute_record_sum(manifest, checksum)}" + else: + hashed_files = None + + # Scan files for editable markers and wheel metadata + files = python_dist.get_paths() + editable = cls._is_record_editable(files) + wheel_file = next((path for path, *_ in files if path.endswith(".dist-info/WHEEL")), None) + if wheel_file: + wheel_details = cls._parse_wheel_file(Path(prefix, wheel_file)) + abi_tags, platform_tags = cls._tags_from_wheel(wheel_details) + else: + abi_tags, platform_tags = (), () + + return cls( + name=record.name, + version=record.version, + python_version=python_version, + python_implementation=python_impl, + files_hash=hashed_files, + python_abi_tags=abi_tags, + python_platform_tags=platform_tags, + editable=editable, ) - with open(json_output.name, "rb") as f: - # We need binary mode because the JSON output might - # contain weird unicode stuff (as part of the project - # description or README). - report = json.loads(f.read()) - os.unlink(json_output.name) - return report + def to_lockfile_line(self) -> list[str]: + if self.url: + return f"{self._line_prefix}{self.url}" + line = ( + f"{self._line_prefix}{self.name}=={self.version}" + f" --python-version {self.python_version}" + f" --implementation {self.python_implementation}" + ) + for abi in self.python_abi_tags: + line += f" --abi {abi}" + for platform in self.python_platform_tags: + line += f" --platform {platform}" + if self.files_hash: + line += f" -- --record-checksum={self.files_hash}" -def compute_record_sum(record_path, algo): - record = Path(record_path).read_text() - lines = [] - for line in record.splitlines(): - path, *_ = line.split(",") + # Here we could try to run a pip --dry-run --report some.json to get the resolved URL + # but it's not guaranteed we get the exact same source so for now we defer to install + # time + + return line + + @staticmethod + def _parse_wheel_file(path) -> dict[str, list[str]]: path = Path(path) - if path.parts[0].endswith(".dist-info") and path.name not in ("METADATA", "WHEEL"): - # we only want to check the metadata and wheel parts of dist-info; everything else - # is not deterministic or useful - continue - if path.parts[0] == ".." and any(part in path.parts for part in ("bin", "lib", "Scripts")): - # entry points are autogenerated and can have different hashes/size depending on prefix - path, *_ = line.split(",") - line = f"{path},," - lines.append(line) - with NamedTemporaryFile("w", delete=False) as tmp: - tmp.write("\n".join(lines)) + if not path.is_file(): + return {} + with open(path) as f: + parsed = HeaderParser().parse(f) + data = {} + for key, value in parsed.items(): + data.setdefault(key, []).append(value) + return data + + @staticmethod + def _tags_from_wheel(data: dict[str, Any]) -> tuple[tuple[str], tuple[str]]: + abi_tags = set() + platform_tags = set() + for tag_str in data.get("Tag", ()): + for tag in parse_tag(tag_str): + if tag.abi != "none": + abi_tags.add(tag.abi) + if tag.platform != "any": + platform_tags.add(tag.platform) + return tuple(abi_tags), tuple(platform_tags) + + @staticmethod + def _is_record_editable(files: tuple[str, str, int]) -> bool: + for path, *_ in files: + path = Path(path) + if "__editable__" in path.stem: + return True + if path.name == "direct_url.json" and path.parent.suffix == ".dist-info": + if path.is_file(): + data = json.loads(path.read_text()) + if data.get("dir_info", {}).get("editable"): + return True + return False + + +def compute_record_sum(manifest: str, algo: str = "sha256") -> str: + """ + Given a RECORD file, compute a hash out of a subset of its sorted contents. + + We skip *.dist-info files other than METADATA and WHEEL. + For non site-packages files, we only keep the path for those than fall in bin, lib and Scripts + because their hash and size might change with path relocation. + + The list of tuples (path, hash, size) is then sorted and written as JSON with no spaces or + indentation. This output is what gets hashed. + """ + manifest = Path(manifest) + if not manifest.is_file(): + return + contents = [] + with open(manifest) as f: + reader = csv_reader(f, delimiter=",", quotechar='"') + for row in reader: + path, hash_, size = row + path = Path(path) + if size: + size = int(size) + if path.parts[0].endswith(".dist-info") and path.name not in ("METADATA", "WHEEL"): + # we only want to check the metadata and wheel parts of dist-info; everything else + # is not deterministic or useful + continue + if path.parts[0] == ".." and any( + part in path.parts for part in ("bin", "lib", "Scripts") + ): + # entry points are autogenerated and can have different hashes/size + # depending on prefix + hash_, size = "", 0 + contents.append((str(path), hash_, size)) try: + with NamedTemporaryFile("w", delete=False) as tmp: + tmp.write(json.dumps(contents, indent=0, separators=(",", ":"))) return compute_sum(tmp.name, algo) finally: os.unlink(tmp.name) From b0cc8cfda93c028c1051042efe2c58f2ae6ce459 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 29 May 2024 14:20:28 +0200 Subject: [PATCH 20/22] mroe refactors --- .gitignore | 3 + conda_pypi/cli/install.py | 145 ++++++++++++++++------------- conda_pypi/cli/list.py | 4 +- conda_pypi/main.py | 188 +++++++++++++++++++++++++++++--------- tests/test_install.py | 3 +- 5 files changed, 230 insertions(+), 113 deletions(-) diff --git a/.gitignore b/.gitignore index 23c92f9..83793bb 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,6 @@ dmypy.json # pixi .pixi/ + +# Used in debugging +explicit.txt diff --git a/conda_pypi/cli/install.py b/conda_pypi/cli/install.py index 914b8d3..7036b9b 100644 --- a/conda_pypi/cli/install.py +++ b/conda_pypi/cli/install.py @@ -1,6 +1,5 @@ from __future__ import annotations -import shlex import sys from logging import getLogger from pathlib import Path @@ -8,49 +7,81 @@ from conda.base.context import context from conda.common.io import Spinner +from conda.exceptions import CondaVerificationError, CondaFileIOError -from ..main import run_pip_install, dry_run_pip_json, compute_record_sum +from ..main import run_pip_install, compute_record_sum, PyPIDistribution from ..utils import get_env_site_packages if TYPE_CHECKING: - from typing import Iterable + from typing import Iterable, Literal log = getLogger(f"conda.{__name__}") -def _prepare_pypi_transaction(pypi_lines) -> dict[str, dict[str, str]]: +def _prepare_pypi_transaction(lines: Iterable[str]) -> dict[str, dict[str, str]]: pkgs = {} - for args in pypi_lines: - args = shlex.split(args) - record_hash = None - if "--" in args: - double_dash_idx = args.index("--") - if double_dash_idx >= 0: - args, extra_args = args[:double_dash_idx], args[double_dash_idx:] - if ( - "--record-checksum" in extra_args - and (hash_idx := extra_args.index("--record-checksum")) > 0 - and extra_args[hash_idx + 1].startswith(("md5:", "sha256:")) - ): - record_hash = extra_args[hash_idx + 1] - else: - for arg in extra_args: - if arg.startswith("--record-checksum="): - record_hash = arg.split("=", 1)[1] - report = dry_run_pip_json(["--no-deps", *args]) - pkg_name = report["install"][0]["metadata"]["name"] - version = report["install"][0]["metadata"]["version"] - pkgs[(pkg_name, version)] = {"url": report["install"][0]["download_info"]["url"]} - if record_hash: - pkgs[(pkg_name, version)]["hash"] = record_hash + for line in lines: + dist = PyPIDistribution.from_lockfile_line(line) + pkgs[(dist.name, dist.version)] = { + "url": dist.find_wheel_url(), + "hashes": dist.record_checksums, + } return pkgs +def _verify_pypi_transaction( + prefix: str, + pkgs: dict[str, dict[str, str]], + on_error: Literal["ignore", "warn", "error"] = "warn", +): + site_packages = get_env_site_packages(prefix) + errors = [] + dist_infos = [path for path in site_packages.glob("*.dist-info") if path.is_dir()] + for (name, version), pkg in pkgs.items(): + norm_name = name.lower().replace("-", "_").replace(".", "_") + dist_info = next( + ( + d + for d in dist_infos + if d.stem.rsplit("-", 1) in ([name, version], [norm_name, version]) + ), + None, + ) + if not dist_info: + errors.append(f"Could not find installation for {name}=={version}") + continue + + expected_hashes = pkg.get("hashes") + if expected_hashes: + found_hashes = compute_record_sum(dist_info / "RECORD", expected_hashes.keys()) + log.info("Verifying %s==%s with %s", name, version, ", ".join(expected_hashes)) + for algo, expected_hash in expected_hashes.items(): + found_hash = found_hashes.get(algo) + if found_hash and expected_hash != found_hash: + msg = ( + "%s checksum for %s==%s didn't match! Expected=%s, found=%s", + algo, + name, + version, + expected_hash, + found_hash, + ) + if on_error == "warn": + log.warning(*msg) + elif on_error == "error": + errors.append(msg[0] % msg[1:]) + else: + log.debug(*msg) + if errors: + errors = "\n- ".join(errors) + raise CondaVerificationError(f"PyPI packages checksum verification failed:\n- {errors}") + + def post_command(command: str) -> int: if command not in ("install", "create"): return 0 - pypi_lines = pypi_lines_from_sys_argv() + pypi_lines = _pypi_lines_from_paths() if not pypi_lines: return 0 @@ -69,43 +100,29 @@ def post_command(command: str) -> int: check=True, ) - if any(pkg.get("hash") for pkg in pkgs.values()): - with Spinner("Verifying PyPI transaction", enabled=not context.quiet, json=context.json): - site_packages = get_env_site_packages(context.target_prefix) - for dist_info in site_packages.glob("*.dist-info"): - if not dist_info.is_dir(): - continue - name, version = dist_info.stem.split("-") - expected_hash = pkgs.get((name, version), {}).get("hash") - if expected_hash: - algo, expected_hash = expected_hash.split(":") - if (dist_info / "RECORD").is_file(): - found_hash = compute_record_sum(dist_info / "RECORD", algo) - if expected_hash != found_hash: - log.warning( - "%s checksum for %s==%s didn't match! Expected=%s, found=%s", - algo, - name, - version, - expected_hash, - found_hash, - ) + with Spinner("Verifying PyPI transaction", enabled=not context.quiet, json=context.json): + on_error_dict = {"disabled": "ignore", "warn": "warn", "enabled": "error"} + on_error = on_error_dict.get(context.safety_checks, "warn") + _verify_pypi_transaction(context.target_prefix, pkgs, on_error=on_error) return 0 -def pypi_lines_from_sys_argv(argv: Iterable[str] | None = None) -> list[str]: - argv = argv or sys.argv - if "--file" not in argv: - return [] - pypi_lines = [] - pypi_prefix = "# pypi: " - pypi_prefix_len = len(pypi_prefix) - for i, arg in enumerate(argv): - if arg == "--file": - pypi_lines += [ - line[pypi_prefix_len:] - for line in Path(argv[i + 1]).read_text().splitlines() - if line.strip().startswith(pypi_prefix) - ] - return pypi_lines +def _pypi_lines_from_paths(paths: Iterable[str] | None = None) -> list[str]: + if paths is None: + file_arg = context.raw_data["cmd_line"].get("file") + if file_arg is None: + return [] + paths = file_arg.value(None) + lines = [] + line_prefix = PyPIDistribution._line_prefix + for path in paths: + path = path.value(None) + try: + with open(path) as f: + for line in f: + if line.startswith(line_prefix): + lines.append(line[len(line_prefix) :]) + except OSError as exc: + raise CondaFileIOError(f"Could not process {path}") from exc + return lines diff --git a/conda_pypi/cli/list.py b/conda_pypi/cli/list.py index ec9fb44..9cf732d 100644 --- a/conda_pypi/cli/list.py +++ b/conda_pypi/cli/list.py @@ -13,8 +13,8 @@ def post_command(command: str): return if "--no-pip" in sys.argv or not cmd_line.get("pip"): return - checksum = "md5" if ("--md5" in sys.argv or cmd_line.get("md5").value(None)) else None - to_print = pypi_lines_for_explicit_lockfile(context.target_prefix, checksum=checksum) + checksums = ("md5",) if ("--md5" in sys.argv or cmd_line.get("md5").value(None)) else None + to_print = pypi_lines_for_explicit_lockfile(context.target_prefix, checksums=checksums) if to_print: sys.stdout.flush() print(f"# The following lines were added by conda-pypi v{__version__}") diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 092985f..71c16b2 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -1,5 +1,6 @@ from __future__ import annotations +import argparse import json import os import shlex @@ -21,6 +22,7 @@ from conda.base.context import context from conda.common.pkg_formats.python import PythonDistribution from conda.core.prefix_data import PrefixData +from conda.exceptions import InvalidVersionSpec from conda.gateways.disk.read import compute_sum from conda.models.enums import PackageType from conda.models.records import PackageRecord @@ -28,6 +30,7 @@ from conda.cli.python_api import run_command from conda.exceptions import CondaError, CondaSystemExit from conda.models.match_spec import MatchSpec +from packaging.requirements import Requirement from packaging.tags import parse_tag from .utils import ( @@ -146,6 +149,8 @@ def run_pip_install( f" stderr:\n{process.stderr}\n" f" stdout:\n{process.stdout}" ) + logger.debug("pip install stdout:\n%s", process.stdout) + logger.debug("pip install stderr:\n%s", process.stderr) return process @@ -202,8 +207,12 @@ def ensure_target_env_has_externally_managed(command: str): def pypi_lines_for_explicit_lockfile( - prefix: Path | str, checksum: Literal["md5", "sha256"] | None = None + prefix: Path | str, checksums: Iterable[Literal["md5", "sha256"]] | None = None ) -> list[str]: + """ + Write pip install pseudo commands for each non-conda-installed Python package in prefix. + See `PyPIDistribution.to_lockfile_line()` for more details. + """ PrefixData._cache_.clear() pd = PrefixData(str(prefix), pip_interop_enabled=True) pd.load() @@ -215,7 +224,7 @@ def pypi_lines_for_explicit_lockfile( if record.package_type != PackageType.VIRTUAL_PYTHON_WHEEL: continue pypi_dist = PyPIDistribution.from_conda_record( - record, python_record, prefix, checksum=checksum + record, python_record, prefix, checksums=checksums ) if pypi_dist.editable: continue @@ -223,7 +232,22 @@ def pypi_lines_for_explicit_lockfile( return lines -def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict[str, Any]: +def dry_run_pip_json( + args: Iterable[str], + ignore_installed: bool = True, + force_reinstall: bool = False, + python_version: str = "", + implementation: str = "", + abi: Iterable[str] = (), + platform: Iterable[str] = (), +) -> dict[str, Any]: + """ + Runs pip in dry-run mode with the goal of obtaining a JSON report that encodes + what would have been done. This is useful to invoke pip as a solver only, or + to obtain the URL of which wheel would have been installed for a particular set of constraints. + + It returns the parsed JSON payload as a dict. + """ # pip can output to stdout via `--report -` (dash), but this # creates issues on Windows due to undecodable characters on some # project descriptions (e.g. charset-normalizer, amusingly), which @@ -238,14 +262,25 @@ def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict "-mpip", "install", "--dry-run", - "--ignore-installed", - *(("--force-reinstall",) if force_reinstall else ()), "--report", json_output.name, "--target", json_output.name + ".dir", # This won't be created - *args, ] + if ignore_installed: + cmd.append("--ignore-installed") + if force_reinstall: + cmd.append("--force-reinstall") + if python_version: + cmd += ["--python-version", python_version] + if implementation: + cmd += ["--implementation", implementation] + for tag in abi: + cmd += ["--abi", tag] + for tag in platform: + cmd += ["--platform", tag] + cmd += args + logger.info("pip dry-run command: %s", cmd) process = run(cmd, capture_output=True, text=True) if process.returncode != 0: raise CondaError( @@ -255,7 +290,8 @@ def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict f" stderr:\n{process.stderr}\n" f" stdout:\n{process.stdout}" ) - + logger.debug("pip dry-run stdout:\n%s", process.stdout) + logger.debug("pip dry-run stderr:\n%s", process.stderr) with open(json_output.name, "rb") as f: # We need binary mode because the JSON output might # contain weird unicode stuff (as part of the project @@ -267,6 +303,7 @@ def dry_run_pip_json(args: Iterable[str], force_reinstall: bool = False) -> dict class PyPIDistribution: _line_prefix = "# pypi: " + _arg_parser = None def __init__( self, @@ -276,7 +313,7 @@ def __init__( python_implementation: str | None = None, python_abi_tags: Iterable[str] = (), python_platform_tags: Iterable[str] = (), - files_hash: str | None = None, + record_checksums: dict[str, str] | None = None, editable: bool = False, ): self.name = name @@ -285,9 +322,9 @@ def __init__( self.python_implementation = python_implementation self.python_abi_tags = python_abi_tags or () self.python_platform_tags = python_platform_tags or () - self.files_hash = files_hash + self.record_checksums = record_checksums or {} self.editable = editable - self.url = None # currently no way to know + self.url = None # currently no way to know, use .find_wheel_url() @classmethod def from_conda_record( @@ -295,7 +332,7 @@ def from_conda_record( record: PackageRecord, python_record: PackageRecord, prefix: str | Path, - checksum: Literal["md5", "sha256"] | None = None, + checksums: Iterable[Literal["md5", "sha256"]] | None = None, ) -> PyPIDistribution: # Locate anchor file sitepackages = get_env_site_packages(prefix) @@ -319,11 +356,11 @@ def from_conda_record( # Find the hash for the RECORD file python_dist = PythonDistribution.init(prefix, str(anchor), python_record.version) - if checksum: + if checksums: manifest = python_dist.manifest_full_path - hashed_files = f"{checksum}:{compute_record_sum(manifest, checksum)}" + record_checksums = compute_record_sum(manifest, checksums) else: - hashed_files = None + record_checksums = None # Scan files for editable markers and wheel metadata files = python_dist.get_paths() @@ -340,13 +377,52 @@ def from_conda_record( version=record.version, python_version=python_version, python_implementation=python_impl, - files_hash=hashed_files, + record_checksums=record_checksums, python_abi_tags=abi_tags, python_platform_tags=platform_tags, editable=editable, ) + @classmethod + def from_lockfile_line(cls, line: str | Iterable[str]): + if isinstance(line, str): + if line.startswith(cls._line_prefix): + line = line[len(cls._line_prefix):] + line = shlex.split(line.strip()) + if cls._arg_parser is None: + cls._arg_parser = cls._build_arg_parser() + args = cls._arg_parser.parse_args(line) + requirement = Requirement(args.spec) + specifiers = list(requirement.specifier) + if len(specifiers) != 1 or specifiers[0].operator != "==": + raise InvalidVersionSpec( + f"{args.spec} is not a valid requirement. " + "PyPI requirements must be exact; i.e. 'name==version'." + ) + pkg_name = requirement.name + version = specifiers[0].version + return cls( + name=pkg_name, + version=version, + python_version=args.python_version, + python_implementation=args.implementation, + python_abi_tags=args.abi, + python_platform_tags=args.platform, + ) + def to_lockfile_line(self) -> list[str]: + """ + Builds a pseudo command-line input for a pip-like interface, with the goal of providing + enough information to retrieve a single wheel or sdist providing the package. The format is: + + ``` + # pypi: [==] [--python-version str] [--implementation str] [--abi str ...] + [--platform str ...] [--record-checksum =] + ``` + + All fields above should be part of the same line. The CLI mimics what `pip` currently + accepts, with the exception of `--record-checksum`, which is a custom addition. + """ if self.url: return f"{self._line_prefix}{self.url}" @@ -359,15 +435,25 @@ def to_lockfile_line(self) -> list[str]: line += f" --abi {abi}" for platform in self.python_platform_tags: line += f" --platform {platform}" - if self.files_hash: - line += f" -- --record-checksum={self.files_hash}" + for algo, checksum in self.record_checksums.items(): + line += f" --record-checksum={algo}:{checksum}" - # Here we could try to run a pip --dry-run --report some.json to get the resolved URL - # but it's not guaranteed we get the exact same source so for now we defer to install - # time + # Here we could invoke self.find_wheel_url() to get the resolved URL but I'm not sure it's + # guaranteed we get the exact same source so for now we defer to install time, which at + # least will give something compatible with the target machine return line + def find_wheel_url(self) -> list[str]: + report = dry_run_pip_json( + ["--no-deps", f"{self.name}=={self.version}"], + python_version=self.python_version, + implementation=self.python_implementation, + abi=self.python_abi_tags, + platform=self.python_platform_tags, + ) + return report["install"][0]["download_info"]["url"] + @staticmethod def _parse_wheel_file(path) -> dict[str, list[str]]: path = Path(path) @@ -405,10 +491,20 @@ def _is_record_editable(files: tuple[str, str, int]) -> bool: return True return False - -def compute_record_sum(manifest: str, algo: str = "sha256") -> str: + @staticmethod + def _build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("spec") + parser.add_argument("--python-version") + parser.add_argument("--implementation") + parser.add_argument("--abi", action="append", default=[]) + parser.add_argument("--platform", action="append", default=[]) + parser.add_argument("--record-checksum", action="append", default=[]) + return parser + +def compute_record_sum(manifest: str, algos: Iterable[str] = ("sha256",)) -> dict[str, str]: """ - Given a RECORD file, compute a hash out of a subset of its sorted contents. + Given a RECORD file, compute hashes out of a subset of its sorted contents. We skip *.dist-info files other than METADATA and WHEEL. For non site-packages files, we only keep the path for those than fall in bin, lib and Scripts @@ -417,32 +513,34 @@ def compute_record_sum(manifest: str, algo: str = "sha256") -> str: The list of tuples (path, hash, size) is then sorted and written as JSON with no spaces or indentation. This output is what gets hashed. """ - manifest = Path(manifest) - if not manifest.is_file(): - return contents = [] - with open(manifest) as f: - reader = csv_reader(f, delimiter=",", quotechar='"') - for row in reader: - path, hash_, size = row - path = Path(path) - if size: - size = int(size) - if path.parts[0].endswith(".dist-info") and path.name not in ("METADATA", "WHEEL"): - # we only want to check the metadata and wheel parts of dist-info; everything else - # is not deterministic or useful - continue - if path.parts[0] == ".." and any( - part in path.parts for part in ("bin", "lib", "Scripts") - ): - # entry points are autogenerated and can have different hashes/size - # depending on prefix - hash_, size = "", 0 - contents.append((str(path), hash_, size)) + try: + with open(manifest) as f: + reader = csv_reader(f, delimiter=",", quotechar='"') + for row in reader: + path, hash_, size = row + path = Path(path) + if size: + size = int(size) + if path.parts[0].endswith(".dist-info") and path.name not in ("METADATA", "WHEEL"): + # we only want to check the metadata and wheel parts of dist-info; everything else + # is not deterministic or useful + continue + if path.parts[0] == ".." and any( + part in path.parts for part in ("bin", "lib", "Scripts") + ): + # entry points are autogenerated and can have different hashes/size + # depending on prefix + hash_, size = "", 0 + contents.append((str(path), hash_, size)) + except OSError as exc: + logger.warning("Could not compute RECORD checksum for %s", manifest) + logger.debug("Could not open %s", manifest, exc_info=exc) + return try: with NamedTemporaryFile("w", delete=False) as tmp: tmp.write(json.dumps(contents, indent=0, separators=(",", ":"))) - return compute_sum(tmp.name, algo) + return {algo: compute_sum(tmp.name, algo) for algo in algos} finally: os.unlink(tmp.name) diff --git a/tests/test_install.py b/tests/test_install.py index 9e4c016..d493511 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -190,8 +190,7 @@ def test_lockfile_roundtrip( print(p.stdout) print(p.stderr, file=sys.stderr) assert p.returncode == 0 - if pure_pip and with_md5: - assert "Verifying PyPI transaction" in p.stdout + assert "Verifying PyPI transaction" in p.stdout out2, err2, rc2 = conda_cli("list", "--explicit", *md5, "--prefix", tmp_path / "env") print(out2) From 674cec6ab59b017e5000fac68227f1a6e0d938a9 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Mon, 3 Jun 2024 10:20:49 +0200 Subject: [PATCH 21/22] fix test --- tests/test_install.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_install.py b/tests/test_install.py index d493511..f1d3cda 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -129,15 +129,9 @@ def test_pyqt( assert conda_spec in out -@pytest.mark.parametrize( - "specs, pure_pip, with_md5", - [ - (("requests",), True, True), - (("requests",), False, False), - (("requests",), True, False), - (("requests",), False, True), - ], -) +@pytest.mark.parametrize("specs", (("requests",),)) +@pytest.mark.parametrize("pure_pip", (True, False)) +@pytest.mark.parametrize("with_md5", (True, False)) def test_lockfile_roundtrip( tmp_path: Path, tmp_env: TmpEnvFixture, @@ -190,7 +184,10 @@ def test_lockfile_roundtrip( print(p.stdout) print(p.stderr, file=sys.stderr) assert p.returncode == 0 - assert "Verifying PyPI transaction" in p.stdout + if pure_pip: + assert "Preparing PyPI transaction" in p.stdout + assert "Executing PyPI transaction" in p.stdout + assert "Verifying PyPI transaction" in p.stdout out2, err2, rc2 = conda_cli("list", "--explicit", *md5, "--prefix", tmp_path / "env") print(out2) From 7cc2bbc4996eb5d8755e4452f1c9327142f7fefc Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 5 Jun 2024 11:36:13 +0200 Subject: [PATCH 22/22] more docs! --- conda_pypi/main.py | 12 ++++---- docs/features.md | 65 +++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 30 +++++++++++++++++++- docs/quickstart.md | 69 +++++++++++++++++++++++++++++++--------------- docs/why.md | 2 +- 5 files changed, 149 insertions(+), 29 deletions(-) create mode 100644 docs/features.md diff --git a/conda_pypi/main.py b/conda_pypi/main.py index 71c16b2..53545ac 100644 --- a/conda_pypi/main.py +++ b/conda_pypi/main.py @@ -422,6 +422,8 @@ def to_lockfile_line(self) -> list[str]: All fields above should be part of the same line. The CLI mimics what `pip` currently accepts, with the exception of `--record-checksum`, which is a custom addition. + + The value of `--record-checksum` is given by `compute_record_sum()`. """ if self.url: return f"{self._line_prefix}{self.url}" @@ -504,13 +506,13 @@ def _build_arg_parser() -> argparse.ArgumentParser: def compute_record_sum(manifest: str, algos: Iterable[str] = ("sha256",)) -> dict[str, str]: """ - Given a RECORD file, compute hashes out of a subset of its sorted contents. + Given a `RECORD` file, compute hashes out of a subset of its sorted contents. - We skip *.dist-info files other than METADATA and WHEEL. - For non site-packages files, we only keep the path for those than fall in bin, lib and Scripts - because their hash and size might change with path relocation. + We skip `*.dist-info` files other than `METADATA` and `WHEEL`. + For non site-packages files, we only keep the path for those than fall in `bin`, `lib` + and `Scripts` because their hash and size might change with path relocation. - The list of tuples (path, hash, size) is then sorted and written as JSON with no spaces or + The list of tuples `(path, hash, size)` is then sorted and written as JSON with no spaces or indentation. This output is what gets hashed. """ contents = [] diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 0000000..ec3fa7c --- /dev/null +++ b/docs/features.md @@ -0,0 +1,65 @@ +# Features + +`conda-pypi` uses the `conda` plugin system to implement several features that make `conda` integrate better with the PyPI ecosystem: + +## The `conda pip` subcommand + +This new subcommand wraps `pip` (and/or other PyPI tools) so you can install PyPI packages (or their conda equivalents) in your conda environment in a safer way. + +The main logic currently works like this: + +1. Collect the PyPI requirements and execute `pip install --dry-run` to obtain a JSON report of "what would have happened". +2. The JSON report is parsed and the resolved dependencies are normalized and mapped to the configured conda channels via different sources (e.g. `cf-graph-countyfair`, `grayskull`, `parselmouth`). +3. The packages that were found on the configured conda channels are installed with `conda`. Those _not_ on conda are installed individually with `pip install --no-deps`. + +:::{admonition} Coming soon +:class: seealso + +Right now we are not disallowing compiled wheels, but we might add options in the future to only allow pure Python wheels via `whl2conda`. +::: + +(pypi-lines)= + +## `conda list` integrations + +`conda` has native support for listing PyPI dependencies as part of `conda list`. However, this is not enabled in all output modes. `conda list --explicit`, used sometimes as a lockfile replacement, does not include any information about the PyPI dependencies. + +We have added a post-command plugin to list PyPI dependencies via `# pypi:` comments. This is currently an experimental, non-standard extension of the file format subject to change. The syntax is: + +``` +# pypi: [==] [--python-version str] [--implementation str] [--abi str ...] [--platform str ...] [--record-checksum =] +``` + +All fields above should be part of the same line. The CLI mimics what `pip` currently accepts (as +of v24.0), with the exception of `--record-checksum`, which is a custom addition. +`--record-checksum` is currently calculated like this: + +1. Given a `RECORD` file, we parse it as a list of 3-tuples: path, hash and size. +2. We skip `*.dist-info` files other than `METADATA` and `WHEEL`. +3. For non site-packages files, we only keep the path for those than fall in `bin`, `lib` + and `Scripts` because their hash and size might change with path relocation. +4. The list of tuples `(path, hash, size)` is then sorted and written as a JSON string with no + spaces or indentation. +5. This is written to a temporary file and then hashed with MD5 or SHA256. + +## `conda install` integrations + +Another post-command plugin is also available to process `@EXPLICIT` lockfiles and search for `# pypi:` lines as discussed above. Again, this is experimental and subject to change. + +## `conda env` integrations + +:::{admonition} Coming soon +:class: seealso + +`environment.yml` files famously allow a `pip` subsection in their `dependencies`. This is handled internally by `conda env` via a `pip` subprocess. We are adding new plugin hooks so `conda-pypi` can handle these in the same way we do with the `conda pip` subcommand. +::: + +(externally-managed)= + +## Environment marker files + +`conda-pypi` adds support for [PEP-668](https://peps.python.org/pep-0668/)'s [`EXTERNALLY-MANAGED`](https://packaging.python.org/en/latest/specifications/externally-managed-environments/) environment marker files. + +This file will tell `pip` and other PyPI installers to not install or remove any packages in that environment, guiding the user towards a safer way of achieving the same result. In our case, the message will let you know that a `conda pip` subcommand is available (see above). + +With this file we mostly want to avoid accidental overwrites that could break your environment. You can still use `pip` directly if you want, but you'll need to add the `--break-system-packages` flag. diff --git a/docs/index.md b/docs/index.md index 0038a52..e08d1b4 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,8 +1,36 @@ # conda-pypi -Welcome to the conda-pypi documentation! +Welcome to the `conda-pypi` documentation! + +`conda-pypi` allows you to run `conda pip install ...` in a safe way, and many other things. + + +::::{grid} 2 + +:::{grid-item-card} 🏡 Getting started +:link: quickstart +:link-type: doc +New to `conda-pypi`? Start here to learn the essentials +::: + +:::{grid-item-card} 💡 Motivation and vision +:link: why +:link-type: doc +Read about why `conda-pypi` exists and when you should use it +::: + +:::{grid-item-card} 🍱 Features +:link: features +:link-type: doc +Overview of what `conda-pypi` can do for you +::: + +:::: ```{toctree} +:hidden: + quickstart why +features ``` diff --git a/docs/quickstart.md b/docs/quickstart.md index f895f9e..551f159 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -10,42 +10,29 @@ conda install -n base conda-pypi ## Basic usage -`conda-pypi` provides several functionalities: +`conda-pypi` provides several {doc}`features`. Some of them are discussed here: -- A `conda pip` subcommand -- A `post_command` hook that will place environment proctection markers +### Safer pip installations -Their usage is discussed below. - -### New environments - -You need to create a new environment with `python` _and_ `pip`, because we will rely on the target `pip` to process the potential PyPI dependencies: - -``` -conda create -n my-python-env python pip -``` - -### Existing environments - -Assuming you have an activated conda environment named `my-python-env` that includes `python` and `pip` installed: +Assuming you have an activated conda environment named `my-python-env` that includes `python` and `pip` installed, and `conda-forge` in your configured channels, you can run `conda pip` like this: ``` conda pip install requests ``` -This will install `requests` from conda, along with all its dependencies, because everything is available. The dependency tree translates one-to-one from PyPI to conda, so there are no issues. +This will install `requests` from conda-forge, along with all its dependencies, because everything is available there. The dependency tree translates one-to-one from PyPI to conda, so there are no issues. ``` conda pip install build ``` -This will install the `python-build` package from conda-forge. Note how `conda pip` knows how to map the different project names. This is done via semi-automated mappings provided by the `grayskull` and `cf-graph-countyfair` projects. +This will install the `python-build` package from conda-forge. Note how `conda pip` knows how to map the different project names. This is done via semi-automated mappings provided by the `grayskull`, `cf-graph-countyfair` and `parselmouth` projects. ``` conda pip install PyQt5 ``` -This will install `pyqt=5` from conda, which also brings `qt=5` separately. This is because `pyqt` on conda _depennds_ on the Qt libraries instead of bundling them in the same package. Again, the `PyQt5 -> pyqt` mapping is handled as expected. +This will install `pyqt=5` from conda, which also brings `qt=5` separately. This is because `pyqt` on conda _depends_ on the Qt libraries instead of bundling them in the same package. Again, the `PyQt5 -> pyqt` mapping is handled as expected. ``` conda pip install ib_insync @@ -61,6 +48,46 @@ conda pip install 5-exercise-upload-to-pypi This package is not available on conda-forge. We will analyze the dependency tree and install all the available ones with `conda`. The rest will be installed with `pip install --no-deps`. +### Lockfiles support + +`conda-pypi` integrates with `conda list --explicit` to add some custom comments so your `@EXPLICIT` lockfiles contain PyPI information. `conda-pypi` also integrates with `conda install` and `conda create` to process these special lines. See more at {ref}`pypi-lines`. + +You can generate these lockfiles with `conda list --explicit --md5`, and they will look like this: + +``` +# This file may be used to create an environment using: +# $ conda create --name --file +# platform: osx-arm64 +@EXPLICIT +https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-h93a5062_5.conda#1bbc659ca658bfd49a481b5ef7a0f40f +https://conda.anaconda.org/conda-forge/osx-arm64/ca-certificates-2024.2.2-hf0a4a13_0.conda#fb416a1795f18dcc5a038bc2dc54edf9 +https://conda.anaconda.org/conda-forge/osx-arm64/libexpat-2.6.2-hebf3989_0.conda#e3cde7cfa87f82f7cb13d482d5e0ad09 +https://conda.anaconda.org/conda-forge/osx-arm64/libffi-3.4.2-h3422bc3_5.tar.bz2#086914b672be056eb70fd4285b6783b6 +https://conda.anaconda.org/conda-forge/osx-arm64/libzlib-1.2.13-h53f4e23_5.conda#1a47f5236db2e06a320ffa0392f81bd8 +https://conda.anaconda.org/conda-forge/osx-arm64/ncurses-6.4.20240210-h078ce10_0.conda#616ae8691e6608527d0071e6766dcb81 +https://conda.anaconda.org/conda-forge/noarch/tzdata-2024a-h0c530f3_0.conda#161081fc7cec0bfda0d86d7cb595f8d8 +https://conda.anaconda.org/conda-forge/osx-arm64/xz-5.2.6-h57fd34a_0.tar.bz2#39c6b54e94014701dd157f4f576ed211 +https://conda.anaconda.org/conda-forge/osx-arm64/libsqlite-3.45.2-h091b4b1_0.conda#9d07427ee5bd9afd1e11ce14368a48d6 +https://conda.anaconda.org/conda-forge/osx-arm64/openssl-3.2.1-h0d3ecfb_1.conda#eb580fb888d93d5d550c557323ac5cee +https://conda.anaconda.org/conda-forge/osx-arm64/readline-8.2-h92ec313_1.conda#8cbb776a2f641b943d413b3e19df71f4 +https://conda.anaconda.org/conda-forge/osx-arm64/tk-8.6.13-h5083fa2_1.conda#b50a57ba89c32b62428b71a875291c9b +https://conda.anaconda.org/conda-forge/osx-arm64/python-3.12.2-hdf0ec26_0_cpython.conda#85e91138ae921a2771f57a50120272bd +https://conda.anaconda.org/conda-forge/noarch/absl-py-2.1.0-pyhd8ed1ab_0.conda#035d1d58677c13ec93122d9eb6b8803b +https://conda.anaconda.org/conda-forge/noarch/setuptools-69.2.0-pyhd8ed1ab_0.conda#da214ecd521a720a9d521c68047682dc +https://conda.anaconda.org/conda-forge/noarch/wheel-0.43.0-pyhd8ed1ab_1.conda#0b5293a157c2b5cd513dd1b03d8d3aae +https://conda.anaconda.org/conda-forge/noarch/pip-24.0-pyhd8ed1ab_0.conda#f586ac1e56c8638b64f9c8122a7b8a67 +# The following lines were added by conda-pypi v0.1.0 +# This is an experimental feature subject to change. Do not use in production. +# pypi: charset-normalizer==3.3.2 --python-version 3.12.2 --implementation cp --abi cp312 --platform macosx_11_0_arm64 --record-checksum=md5:a88a07f3a23748b3d78b24ca3812e7d8 +# pypi: certifi==2024.2.2 --python-version 3.12.2 --implementation cp --record-checksum=md5:1c186605aa7d0c050cf4ef147fcf750d +# pypi: tf-slim==1.1.0 --python-version 3.12.2 --implementation cp --record-checksum=md5:96c65c0d90cd8c93f3bbe22ee34190c5 +# pypi: aaargh==0.7.1 --python-version 3.12.2 --implementation cp --record-checksum=md5:55f5aa1765064955792866812afdef6f +# pypi: requests==2.32.2 --python-version 3.12.2 --implementation cp --record-checksum=md5:d7e8849718b3ffb565fd3cbe2575ea97 +# pypi: 5-exercise-upload-to-pypi==1.2 --python-version 3.12.2 --implementation cp --record-checksum=md5:c96a1cd6037f6e3b659e2139b0839c97 +# pypi: idna==3.7 --python-version 3.12.2 --implementation cp --record-checksum=md5:5b2f9f2c52705a9b1e32818f1b387356 +# pypi: urllib3==2.2.1 --python-version 3.12.2 --implementation cp --record-checksum=md5:1bd9312a95c73a644f721ca96c9d8b45 +``` + ### Environment protection `conda-pypi` ships a special file, `EXTERNALLY-MANAGED`, that will be installed in: @@ -69,6 +96,4 @@ This package is not available on conda-forge. We will analyze the dependency tre - All new environments that include `pip`. - Existing environments that `pip`, but only after running a conda command on them. -This file is designed after [PEP668](https://peps.python.org/pep-0668/). You can read more about in [Externally Managed Environments at packaging.python.org](https://packaging.python.org/en/latest/specifications/externally-managed-environments/). - -Essentially, the presence of this file in a given environment will prevent users from using `pip` directly on them. An [informative error message](https://github.com/jaimergp/conda-pip/blob/main/conda_pypi/data/EXTERNALLY-MANAGED) is provided instead. +More details at {ref}`externally-managed`. diff --git a/docs/why.md b/docs/why.md index 65d1e22..c7be388 100644 --- a/docs/why.md +++ b/docs/why.md @@ -54,4 +54,4 @@ Are we expecting you to do all that manually? Of course not! This is what `conda ## Expected behavior -Refer to the [Quickstart guide](quickstart.md). +Refer to the [Quick start guide](quickstart.md).