diff --git a/conda_lock/src_parser/__init__.py b/conda_lock/src_parser/__init__.py index 8e9506ef..d3cd0543 100644 --- a/conda_lock/src_parser/__init__.py +++ b/conda_lock/src_parser/__init__.py @@ -3,10 +3,14 @@ from typing import AbstractSet, List, Optional, Sequence +from conda_lock.common import ordered_union from conda_lock.models.channel import Channel from conda_lock.models.lock_spec import Dependency, LockSpecification from conda_lock.src_parser.aggregation import aggregate_lock_specs -from conda_lock.src_parser.environment_yaml import parse_environment_file +from conda_lock.src_parser.environment_yaml import ( + parse_environment_file, + parse_platforms_from_env_file, +) from conda_lock.src_parser.meta_yaml import parse_meta_yaml_file from conda_lock.src_parser.pyproject_toml import parse_pyproject_toml from conda_lock.virtual_package import FakeRepoData @@ -18,9 +22,30 @@ logger = logging.getLogger(__name__) +def _parse_platforms_from_srcs(src_files: List[pathlib.Path]) -> List[str]: + """ + Parse a sequence of dependency specifications from source files + + Parameters + ---------- + src_files : + Files to parse for dependencies + """ + all_file_platforms: List[List[str]] = [] + for src_file in src_files: + if src_file.name == "meta.yaml": + continue + elif src_file.name == "pyproject.toml": + all_file_platforms.append(parse_pyproject_toml(src_file).platforms) + else: + all_file_platforms.append(parse_platforms_from_env_file(src_file)) + + return ordered_union(all_file_platforms) + + def _parse_source_files( src_files: List[pathlib.Path], - platform_overrides: Optional[Sequence[str]], + platforms: List[str], ) -> List[LockSpecification]: """ Parse a sequence of dependency specifications from source files @@ -29,27 +54,17 @@ def _parse_source_files( ---------- src_files : Files to parse for dependencies - platform_overrides : + platforms : Target platforms to render environment.yaml and meta.yaml files for """ desired_envs: List[LockSpecification] = [] for src_file in src_files: if src_file.name == "meta.yaml": - desired_envs.append( - parse_meta_yaml_file( - src_file, list(platform_overrides or DEFAULT_PLATFORMS) - ) - ) + desired_envs.append(parse_meta_yaml_file(src_file, platforms)) elif src_file.name == "pyproject.toml": desired_envs.append(parse_pyproject_toml(src_file)) else: - desired_envs.append( - parse_environment_file( - src_file, - platform_overrides, - default_platforms=DEFAULT_PLATFORMS, - ) - ) + desired_envs.append(parse_environment_file(src_file, platforms)) return desired_envs @@ -62,20 +77,22 @@ def make_lock_spec( required_categories: Optional[AbstractSet[str]] = None, ) -> LockSpecification: """Generate the lockfile specs from a set of input src_files. If required_categories is set filter out specs that do not match those""" - lock_specs = _parse_source_files( - src_files=src_files, platform_overrides=platform_overrides - ) + platforms = ( + list(platform_overrides) + if platform_overrides + else _parse_platforms_from_srcs(src_files) + ) or DEFAULT_PLATFORMS + + lock_specs = _parse_source_files(src_files, platforms) lock_spec = aggregate_lock_specs(lock_specs) lock_spec.virtual_package_repo = virtual_package_repo + lock_spec.platforms = platforms lock_spec.channels = ( [Channel.from_string(co) for co in channel_overrides] if channel_overrides else lock_spec.channels ) - lock_spec.platforms = ( - list(platform_overrides) if platform_overrides else lock_spec.platforms - ) or list(DEFAULT_PLATFORMS) if required_categories is not None: diff --git a/conda_lock/src_parser/environment_yaml.py b/conda_lock/src_parser/environment_yaml.py index 1da9e3cc..5a853d6e 100644 --- a/conda_lock/src_parser/environment_yaml.py +++ b/conda_lock/src_parser/environment_yaml.py @@ -2,7 +2,7 @@ import re import sys -from typing import List, Optional, Sequence, Tuple +from typing import List, Tuple import yaml @@ -96,11 +96,23 @@ def _parse_environment_file_for_platform( ) +def parse_platforms_from_env_file(environment_file: pathlib.Path) -> List[str]: + """ + Parse the list of platforms from an environment-yaml file + """ + if not environment_file.exists(): + raise FileNotFoundError(f"{environment_file} not found") + + with environment_file.open("r") as fo: + content = fo.read() + env_yaml_data = yaml.safe_load(content) + + return env_yaml_data.get("platforms", []) + + def parse_environment_file( environment_file: pathlib.Path, - given_platforms: Optional[Sequence[str]], - *, - default_platforms: List[str] = [], + platforms: List[str], ) -> LockSpecification: """Parse a simple environment-yaml file for dependencies assuming the target platforms. @@ -114,15 +126,6 @@ def parse_environment_file( with environment_file.open("r") as fo: content = fo.read() - env_yaml_data = yaml.safe_load(content) - - # Get list of platforms from the input file - yaml_platforms: Optional[List[str]] = env_yaml_data.get("platforms") - # Final list of platforms is the following order of priority - # 1) List Passed in via the -p flag (if any given) - # 2) List From the YAML File (if specified) - # 3) Default List of Platforms to Render - platforms = list(given_platforms or yaml_platforms or default_platforms) # Parse with selectors for each target platform spec = aggregate_lock_specs( diff --git a/tests/test-multi-sources/dev.yml b/tests/test-multi-sources/dev.yml new file mode 100644 index 00000000..473a9ad7 --- /dev/null +++ b/tests/test-multi-sources/dev.yml @@ -0,0 +1,2 @@ +dependencies: + - pytest diff --git a/tests/test-multi-sources/main.yml b/tests/test-multi-sources/main.yml new file mode 100644 index 00000000..e1a68acd --- /dev/null +++ b/tests/test-multi-sources/main.yml @@ -0,0 +1,6 @@ +dependencies: + - python <3.11 +platforms: + - osx-arm64 + - osx-64 + - linux-64 diff --git a/tests/test-multi-sources/pyproject.toml b/tests/test-multi-sources/pyproject.toml new file mode 100644 index 00000000..06246cba --- /dev/null +++ b/tests/test-multi-sources/pyproject.toml @@ -0,0 +1,18 @@ +[tool.poetry] +name = "conda-lock-test-poetry" +version = "0.0.1" +description = "" +authors = ["conda-lock"] + +[tool.poetry.dependencies] +pandas = "^1.5.0" + +[build-system] +requires = ["poetry>=0.12"] +build-backend = "poetry.masonry.api" + +[tool.conda-lock] +platforms = [ + "win-64", + "osx-arm64" +] diff --git a/tests/test_conda_lock.py b/tests/test_conda_lock.py index 64aff8a1..943ac878 100644 --- a/tests/test_conda_lock.py +++ b/tests/test_conda_lock.py @@ -67,10 +67,14 @@ from conda_lock.src_parser import ( DEFAULT_PLATFORMS, LockSpecification, + _parse_platforms_from_srcs, parse_meta_yaml_file, ) from conda_lock.src_parser.aggregation import aggregate_lock_specs -from conda_lock.src_parser.environment_yaml import parse_environment_file +from conda_lock.src_parser.environment_yaml import ( + parse_environment_file, + parse_platforms_from_env_file, +) from conda_lock.src_parser.pyproject_toml import ( parse_pyproject_toml, poetry_version_to_conda_version, @@ -212,6 +216,12 @@ def git_metadata_zlib_environment(tmp_path: Path): return clone_test_dir("zlib", tmp_path).joinpath("environment.yml") +@pytest.fixture +def multi_source_env(tmp_path: Path): + f = clone_test_dir("test-multi-sources", tmp_path) + return [f.joinpath("main.yml"), f.joinpath("pyproject.toml"), f.joinpath("dev.yml")] + + @pytest.fixture( scope="function", params=[ @@ -360,7 +370,8 @@ def test_parse_environment_file_with_pip(pip_environment: Path): def test_parse_env_file_with_filters_no_args(filter_conda_environment: Path): - res = parse_environment_file(filter_conda_environment, None) + platforms = parse_platforms_from_env_file(filter_conda_environment) + res = parse_environment_file(filter_conda_environment, platforms) assert all(x in res.platforms for x in ["osx-arm64", "osx-64", "linux-64"]) assert res.channels == [Channel.from_string("conda-forge")] @@ -423,6 +434,11 @@ def test_parse_env_file_with_filters_defaults(filter_conda_environment: Path): ) +def test_parse_platforms_from_multi_sources(multi_source_env): + platforms = _parse_platforms_from_srcs(multi_source_env) + assert platforms == ["osx-arm64", "osx-64", "linux-64", "win-64"] + + def test_choose_wheel() -> None: solution = solve_pypi( {