Merge pull request #588 from ianpye/bugfix/lookup-flag

Proposed fix for #579

conda_lock/lookup.py

@@ -1,4 +1,5 @@
 from functools import cached_property
+from pathlib import Path
 from typing import Dict
 
 import requests
@@ -26,15 +27,32 @@ def mapping_url(self) -> str:
 
     @mapping_url.setter
     def mapping_url(self, value: str) -> None:
-        del self.pypi_lookup
-        del self.conda_lookup
-        self._mapping_url = value
+        if self._mapping_url != value:
+            self._mapping_url = value
+            # Invalidate cache
+            try:
+                del self.pypi_lookup
+            except AttributeError:
+                pass
+            try:
+                del self.conda_lookup
+            except AttributeError:
+                pass
 
     @cached_property
     def pypi_lookup(self) -> Dict[NormalizedName, MappingEntry]:
-        res = requests.get(self._mapping_url)
-        res.raise_for_status()
-        lookup = yaml.safe_load(res.content)
+        url = self.mapping_url
+        if url.startswith("http://") or url.startswith("https://"):
+            res = requests.get(self._mapping_url)
+            res.raise_for_status()
+            content = res.content
+        else:
+            if url.startswith("file://"):
+                path = url[len("file://") :]
+            else:
+                path = url
+            content = Path(path).read_bytes()
+        lookup = yaml.safe_load(content)
         # lowercase and kebabcase the pypi names
         assert lookup is not None
         lookup = {canonicalize_name(k): v for k, v in lookup.items()}

tests/test-lookup/emoji-to-python-dateutil-lookup.yml

@@ -0,0 +1,8 @@
+# Trick conda-lock into thinking that the emoji Conda package provides
+# python-dateutil as a pip dependency
+
+python-dateutil:
+  conda_name: emoji
+  import_name: emoji
+  mapping_source: conda-lock-test-suite
+  pypi_name: python-dateutil

tests/test-lookup/empty-lookup.yml

@@ -0,0 +1 @@
+{}

tests/test-lookup/environment.yml

@@ -0,0 +1,14 @@
+channels:
+- conda-forge
+- nodefaults
+platforms:
+- linux-64
+dependencies:
+# The emoji package has no dependencies.
+- emoji==2.8.0
+- pip
+- pip:
+  # Arrow's dependencies are python-dateutil >=2.7.0 and types-python-dateutil >=2.8.10.
+  # Note that a version 2.8.0 exists of python-dateutil. This makes it possible
+  # to add an entry to the lookup table that maps emoji to python-dateutil.
+  - arrow==1.3.0

tests/test_conda_lock.py

@@ -66,6 +66,7 @@
     LockedDependency,
     MetadataOption,
 )
+from conda_lock.lookup import _LookupLoader
 from conda_lock.models.channel import Channel
 from conda_lock.models.lock_spec import Dependency, VCSDependency, VersionedDependency
 from conda_lock.models.pip_repository import PipRepository
@@ -2425,6 +2426,90 @@ def run_install():
         run_install()
 
 
+def test_lookup_sources():
+    # Test that the lookup can be read from a file:// URL
+    lookup = (
+        Path(__file__).parent / "test-lookup" / "emoji-to-python-dateutil-lookup.yml"
+    )
+    url = f"file://{lookup.absolute()}"
+    LOOKUP_OBJECT = _LookupLoader()
+    LOOKUP_OBJECT.mapping_url = url
+    assert LOOKUP_OBJECT.conda_lookup["emoji"]["pypi_name"] == "python-dateutil"
+
+    # Test that the lookup can be read from a straight filename
+    url = str(lookup.absolute())
+    LOOKUP_OBJECT = _LookupLoader()
+    LOOKUP_OBJECT.mapping_url = url
+    assert LOOKUP_OBJECT.conda_lookup["emoji"]["pypi_name"] == "python-dateutil"
+
+    # Test that the default remote lookup contains expected nontrivial mappings
+    LOOKUP_OBJECT = _LookupLoader()
+    assert LOOKUP_OBJECT.conda_lookup["python-build"]["pypi_name"] == "build"
+
+
+@pytest.fixture
+def lookup_environment(tmp_path: Path):
+    return clone_test_dir("test-lookup", tmp_path).joinpath("environment.yml")
+
+
+@pytest.mark.parametrize(
+    "lookup_source", ["emoji-to-python-dateutil-lookup.yml", "empty-lookup.yml"]
+)
+def test_lookup(
+    lookup_environment: Path,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys: pytest.CaptureFixture[str],
+    lookup_source: str,
+):
+    """We test that the lookup table is being used to convert conda package names into
+    pypi package names. We verify this by comparing the results from using two
+    different lookup tables.
+
+    The pip solver runs after the conda solver. The pip solver needs to know which
+    packages are already installed by conda. The lookup table is used to convert conda
+    package names into pypi package names.
+
+    We test two cases:
+    1. The lookup table is empty. In this case, the conda package names are converted
+    directly into pypi package names. As long as there are no discrepancies between
+    conda and pypi package names, this gives expected results.
+    2. The lookup table maps emoji to python-dateutil. Arrow is installed as a pip
+    package and has python-dateutil as a dependency. Due to this lookup table, the
+    pip solver should believe that the dependency is already satisfied and not add it.
+    """
+    cwd = lookup_environment.parent
+    monkeypatch.chdir(cwd)
+    lookup_filename = str((cwd / lookup_source).absolute())
+    with capsys.disabled():
+        from click.testing import CliRunner, Result
+
+        runner = CliRunner(mix_stderr=False)
+        result: Result = runner.invoke(
+            main,
+            ["lock", "--pypi_to_conda_lookup_file", lookup_filename],
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 0
+
+    lockfile = cwd / DEFAULT_LOCKFILE_NAME
+    assert lockfile.is_file()
+    lockfile_content = parse_conda_lock_file(lockfile)
+    installed_packages = {p.name for p in lockfile_content.package}
+    assert "emoji" in installed_packages
+    assert "arrow" in installed_packages
+    assert "types-python-dateutil" in installed_packages
+    if lookup_source == "empty-lookup.yml":
+        # If the lookup table is empty, then conda package names are converted
+        # directly into pypi package names. Arrow depends on python-dateutil, so
+        # it should be installed.
+        assert "python-dateutil" in installed_packages
+    else:
+        # The nonempty lookup table maps emoji to python-dateutil. Thus the pip
+        # solver should believe that the dependency is already satisfied and not
+        # add it as a pip dependency.
+        assert "python-dateutil" not in installed_packages
+
+
 def test_extract_json_object():
     """It should remove all the characters after the last }"""
     assert extract_json_object(' ^[0m {"key1": true } ^[0m') == '{"key1": true }'