Merge pull request #291 from srilman/gh-134-os-filters

conda_lock/conda_lock.py

@@ -252,7 +252,7 @@ def make_lock_spec(
 ) -> LockSpecification:
     """Generate the lockfile specs from a set of input src_files.  If required_categories is set filter out specs that do not match those"""
     lock_specs = parse_source_files(
-        src_files=src_files, platform_overrides=platform_overrides or DEFAULT_PLATFORMS
+        src_files=src_files, platform_overrides=platform_overrides
     )
 
     lock_spec = aggregate_lock_specs(lock_specs)
@@ -857,7 +857,7 @@ def create_lockfile_from_spec(
 
 def parse_source_files(
     src_files: List[pathlib.Path],
-    platform_overrides: Sequence[str],
+    platform_overrides: Optional[Sequence[str]],
 ) -> List[LockSpecification]:
     """
     Parse a sequence of dependency specifications from source files
@@ -867,19 +867,26 @@ def parse_source_files(
     src_files :
         Files to parse for dependencies
     platform_overrides :
-        Target platforms to render meta.yaml files for
+        Target platforms to render environment.yaml and meta.yaml files for
     """
     desired_envs: List[LockSpecification] = []
     for src_file in src_files:
         if src_file.name == "meta.yaml":
             desired_envs.append(
-                parse_meta_yaml_file(src_file, list(platform_overrides))
+                parse_meta_yaml_file(
+                    src_file, list(platform_overrides or DEFAULT_PLATFORMS)
+                )
             )
         elif src_file.name == "pyproject.toml":
             desired_envs.append(parse_pyproject_toml(src_file))
         else:
             desired_envs.append(
-                parse_environment_file(src_file, pip_support=PIP_SUPPORT)
+                parse_environment_file(
+                    src_file,
+                    platform_overrides,
+                    default_platforms=DEFAULT_PLATFORMS,
+                    pip_support=PIP_SUPPORT,
+                )
             )
     return desired_envs
 

conda_lock/src_parser/environment_yaml.py

@@ -2,11 +2,11 @@
 import re
 import sys
 
-from typing import List, Tuple
+from typing import List, Optional, Sequence, Tuple
 
 import yaml
 
-from conda_lock.src_parser import Dependency, LockSpecification
+from conda_lock.src_parser import Dependency, LockSpecification, aggregate_lock_specs
 from conda_lock.src_parser.conda_common import conda_spec_to_versioned_dep
 from conda_lock.src_parser.selectors import filter_platform_selectors
 
@@ -25,11 +25,16 @@ def parse_conda_requirement(req: str) -> Tuple[str, str]:
         raise ValueError(f"Can't parse conda spec from '{req}'")
 
 
-def parse_environment_file(
-    environment_file: pathlib.Path, *, pip_support: bool = False
+def _parse_environment_file_for_platform(
+    environment_file: pathlib.Path,
+    content: str,
+    platform: str,
+    *,
+    pip_support: bool = False,
 ) -> LockSpecification:
     """
-    Parse dependencies from a conda environment specification
+    Parse dependencies from a conda environment specification for an
+    assumed target platform.
 
     Parameters
     ----------
@@ -38,35 +43,29 @@ def parse_environment_file(
     pip_support :
         Emit dependencies in pip section of environment.yml. If False, print a
         warning and ignore pip dependencies.
-
+    platform :
+        Target platform to use when parsing selectors to filter lines
     """
-    dependencies: List[Dependency] = []
-    if not environment_file.exists():
-        raise FileNotFoundError(f"{environment_file} not found")
-
-    with environment_file.open("r") as fo:
-        content = fo.read()
-        # TODO: improve this call since we *SHOULD* pass the right platform here
-        filtered_content = "\n".join(filter_platform_selectors(content, platform=""))
-        assert yaml.safe_load(filtered_content) == yaml.safe_load(
-            content
-        ), "selectors are temporarily gone"
+    filtered_content = "\n".join(filter_platform_selectors(content, platform=platform))
+    env_yaml_data = yaml.safe_load(filtered_content)
 
-        env_yaml_data = yaml.safe_load(filtered_content)
     specs = env_yaml_data["dependencies"]
-    channels = env_yaml_data.get("channels", [])
+    channels: List[str] = env_yaml_data.get("channels", [])
 
     # These extension fields are nonstandard
-    platforms = env_yaml_data.get("platforms", [])
-    category = env_yaml_data.get("category") or "main"
+    platforms: List[str] = env_yaml_data.get("platforms", [])
+    category: str = env_yaml_data.get("category") or "main"
 
     # Split out any sub spec sections from the dependencies mapping
     mapping_specs = [x for x in specs if not isinstance(x, str)]
     specs = [x for x in specs if isinstance(x, str)]
 
+    dependencies: List[Dependency] = []
     for spec in specs:
         vdep = conda_spec_to_versioned_dep(spec, category)
+        vdep.selectors.platform = [platform]
         dependencies.append(vdep)
+
     for mapping_spec in mapping_specs:
         if "pip" in mapping_spec:
             if pip_support:
@@ -105,7 +104,59 @@ def parse_environment_file(
 
     return LockSpecification(
         dependencies=dependencies,
-        channels=channels,
+        channels=channels,  # type: ignore
         platforms=platforms,
         sources=[environment_file],
     )
+
+
+def parse_environment_file(
+    environment_file: pathlib.Path,
+    given_platforms: Optional[Sequence[str]],
+    *,
+    default_platforms: List[str] = [],
+    pip_support: bool = False,
+) -> LockSpecification:
+    """Parse a simple environment-yaml file for dependencies assuming the target platforms.
+
+    * This will emit one dependency set per target platform. These may differ
+      if the dependencies depend on platform selectors.
+    * This does not support multi-output files and will ignore all lines with
+      selectors other than platform.
+    """
+    if not environment_file.exists():
+        raise FileNotFoundError(f"{environment_file} not found")
+
+    with environment_file.open("r") as fo:
+        content = fo.read()
+        env_yaml_data = yaml.safe_load(content)
+
+    # Get list of platforms from the input file
+    yaml_platforms: Optional[List[str]] = env_yaml_data.get("platforms")
+    # Final list of platforms is the following order of priority
+    # 1) List Passed in via the -p flag (if any given)
+    # 2) List From the YAML File (if specified)
+    # 3) Default List of Platforms to Render
+    platforms = list(given_platforms or yaml_platforms or default_platforms)
+
+    # Parse with selectors for each target platform
+    spec = aggregate_lock_specs(
+        [
+            _parse_environment_file_for_platform(
+                environment_file, content, platform, pip_support=pip_support
+            )
+            for platform in platforms
+        ]
+    )
+
+    # Remove platform selectors if they apply to all targets
+    for dep in spec.dependencies:
+        if dep.selectors.platform == platforms:
+            dep.selectors.platform = None
+
+    # Use the list of rendered platforms for the output spec only if
+    # there is a dependency that is not used on all platforms.
+    # This is unlike meta.yaml because environment-yaml files can contain an
+    # internal list of platforms, which should be used as long as it
+    spec.platforms = platforms
+    return spec

conda_lock/src_parser/selectors.py

@@ -1,20 +1,26 @@
 import logging
 import re
 
-from typing import Iterator
+from typing import Iterator, Optional
 
 
 logger = logging.getLogger(__name__)
 
 
-def filter_platform_selectors(content: str, platform: str) -> Iterator[str]:
+def filter_platform_selectors(
+    content: str, platform: Optional[str] = None
+) -> Iterator[str]:
     """ """
     # we support a very limited set of selectors that adhere to platform only
+    # https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#preprocessing-selectors
+
     platform_sel = {
         "linux-64": {"linux64", "unix", "linux"},
         "linux-aarch64": {"aarch64", "unix", "linux"},
         "linux-ppc64le": {"ppc64le", "unix", "linux"},
-        "osx-64": {"osx", "osx64", "unix"},
+        # "osx64" is a selector unique to conda-build referring to
+        # platforms on macOS and the Python architecture is x86-64
+        "osx-64": {"osx64", "osx", "unix"},
         "osx-arm64": {"arm64", "osx", "unix"},
         "win-64": {"win", "win64"},
     }
@@ -25,9 +31,9 @@ def filter_platform_selectors(content: str, platform: str) -> Iterator[str]:
         if line.lstrip().startswith("#"):
             continue
         m = sel_pat.match(line)
-        if m:
+        if platform and m:
             cond = m.group(3)
-            if platform and (cond in platform_sel[platform]):
+            if cond in platform_sel[platform]:
                 yield line
             else:
                 logger.warning(

docs/src_environment_yml.md

@@ -39,7 +39,7 @@ override the values in the environment specification. If neither `platforms` nor
 
 ### Categories
 
-You can may wish to split your dependencies into separate files for better
+You may wish to split your dependencies into separate files for better
 organization, e.g. a `environment.yml` for production dependencies and a
 `dev-environment.yml` for development dependencies. You can assign all the
 dependencies parsed from a single file to a category using the (nonstandard)
@@ -62,4 +62,27 @@ These can be used in a [compound specification](/compound_specification) as foll
 conda-lock --file environment.yml --file dev-environment.yml
 ```
 
-[envyaml]: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually
+### Preprocessing Selectors
+
+You may use preprocessing selectors supported by conda-build `meta.yaml` files to specify platform-specific dependencies.
+
+```{.yaml title="environment.yml"}
+channels:
+  - conda-forge
+dependencies:
+  - python=3.9
+  - pywin32 # [win]
+platforms:
+  - linux-64
+  - win-64
+```
+
+There are currently some limitations to selectors to be aware of:
+- Only OS-specific selectors are currently supported. See Conda's [documentation][selectors] for the list of supported selectors. Selectors related to Python or Numpy versions are not supported
+- conda-lock supports an additional unique selector `osx64`. It is true if the platform is macOS and the Python architecture is 64-bit and uses x86.
+- `not`, `and`, and `or` clauses inside of selectors are not supported
+- Comparison operators (`==`, `>`, `<`, etc) are not supported
+
+
+[envyaml]: https://docs.conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#create-env-file-manually
+[selectors]: https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html#preprocessing-selectors

docs/src_meta_yaml.md

@@ -57,6 +57,15 @@ is being built from support them).  This can be disabled easily
 conda-lock --no-dev-dependencies --file meta.yaml
 ```
 
+## Preprocessing Selectors
+
+You may use preprocessing selectors, but there are currently some limitations to be aware of:
+- Only OS-specific selectors are currently supported. See Conda's [documentation][selectors] for the list of supported selectors. Selectors related to Python or Numpy versions are not supported
+- conda-lock supports an additional unique selector `osx64`. It is true if the platform is macOS and the Python architecture is 64-bit and uses x86.
+- `not`, `and`, and `or` clauses inside of selectors are not supported
+- Comparison operators (`==`, `>`, `<`, etc) are not supported
+
+
 ## Extensions
 
 ### Channel specification

tests/test-env-filter-platform/environment.yml

@@ -0,0 +1,11 @@
+channels:
+  - conda-forge
+dependencies:
+  - python <3.11
+  - clang_osx-arm64  # [arm64]
+  - clang_osx-64  # [osx64]
+  - gcc_linux-64>=6  # [linux64]
+platforms:
+  - osx-arm64
+  - osx-64
+  - linux-64