Micromamba for ci (#205)

* add ci envs

* micromamba for unit test envs

* use micromamba run to execute e2e test

* fix env directory in workflow yaml

.github/workflows/unit-test.yaml

@@ -12,43 +12,47 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        environment-name: [
+          "py3.10-ecoscope1.8.2",
+          "py3.11-ecoscope1.8.2",
+          "py3.12-ecoscope1.8.2",
+        ]
 
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: Setup Micromamba
+        uses: mamba-org/setup-micromamba@v1
         with:
-          python-version: ${{ matrix.python-version }}
+          environment-file: ./ci/envs/${{ matrix.environment-name }}.yml
+          cache-environment: true
+          cache-downloads: true
+          init-shell: bash
 
-      - uses: actions/cache@v4
-        id: restore-cache
-        with:
-          key: "venv-${{ matrix.python-version }}-${{ hashFiles('pyproject.toml') }}"
-          path: |
-            .venv/
-
-      # If python patch version changes, or pyproject.toml gets updated,
-      # the cache lookup will miss, and the  venv needs to be recreated.
-      - name: If no cache hit, recreate venv
-        if: "steps.restore-cache.outputs.cache-hit == false"
+      - name: Install pip dependencies and our package
+        shell: bash -leo pipefail {0}
         run: |
-          python -m venv .venv
-          .venv/bin/python -m pip install --upgrade pip
-          .venv/bin/python -m pip install -e ".[test]"
+          python -m pip install -U pip
+          python -m pip install -e ".[test,lithops]" --config-settings editable_mode=compat
 
       - name: Run doctests
+        shell: bash -leo pipefail {0}
         run: |
-          .venv/bin/python -m pytest -v ecoscope_workflows/ --doctest-modules \
+          python -m pytest -v ecoscope_workflows/ --doctest-modules \
           --ignore=ecoscope_workflows/visualize.py
 
+      - name: Export shell name and conda env name to env
+        run: |
+          echo 'SHELL="bash -leo pipefail"' >> $GITHUB_ENV
+          echo "CONDA_ENV_NAME=${{ matrix.environment-name }}" >> $GITHUB_ENV
+
       - name: Test with pytest
         env:
           EARTHRANGER_SERVER: ${{ secrets.EARTHRANGER_SERVER }}
           EARTHRANGER_USERNAME: ${{ secrets.EARTHRANGER_USERNAME }}
           EARTHRANGER_PASSWORD: ${{ secrets.EARTHRANGER_PASSWORD }}
+        shell: bash -leo pipefail {0}
         run: |
-          .venv/bin/python -m pytest -n auto tests -vvv
+          python -m pytest tests -vvv
 
     # - name: Upload Coverage to Codecov
     #   uses: codecov/codecov-action@v2

ci/envs/py3.10-ecoscope1.8.2.yml

@@ -0,0 +1,11 @@
+name: py3.10-ecoscope1.8.2
+channels:
+  - conda-forge
+dependencies:
+  - python==3.10.12
+  - geopandas<=0.14.2
+  - git
+  - numpy<2
+  - pip
+  - pip:
+    - ecoscope[analysis,mapping,plotting] @ git+https://github.com/wildlife-dynamics/[email protected]

ci/envs/py3.11-ecoscope1.8.2.yml

@@ -0,0 +1,11 @@
+name: py3.11-ecoscope1.8.2
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+  - geopandas<=0.14.2
+  - git
+  - numpy<2
+  - pip
+  - pip:
+    - ecoscope[analysis,mapping,plotting] @ git+https://github.com/wildlife-dynamics/[email protected]

ci/envs/py3.12-ecoscope1.8.2.yml

@@ -0,0 +1,11 @@
+name: py3.12-ecoscope1.8.2
+channels:
+  - conda-forge
+dependencies:
+  - python=3.12
+  - geopandas<=0.14.2
+  - git
+  - numpy<2
+  - pip
+  - pip:
+    - ecoscope[analysis,mapping,plotting] @ git+https://github.com/wildlife-dynamics/[email protected]

tests/test_examples.py

@@ -164,17 +164,42 @@ def test_end_to_end(end_to_end: EndToEndFixture, tmp_path: Path):
     with open(script_outpath, mode="w") as f:
         f.write(script)
 
-    cmd = [
-        sys.executable,
-        "-W",
-        "ignore",  # in testing context warnings are added; exclude them from stdout
-        script_outpath.as_posix(),
-        "--config-file",
-        end_to_end.param_path.as_posix(),
-    ]
+    exe = (
+        # workaround for https://github.com/mamba-org/mamba/issues/2577
+        f"{os.environ['MAMBA_EXE']} run -n {os.environ['CONDA_ENV_NAME']} python"
+        if "mamba" in sys.executable
+        else sys.executable
+    )
+    cmd = " ".join(
+        [
+            os.environ.get("SHELL", "/bin/sh").replace('"', "").replace("'", ""),
+            "-c",
+            f"'{exe}",
+            "-W",
+            "ignore",  # in testing context warnings are added; exclude them from stdout
+            script_outpath.as_posix(),
+            "--config-file",
+            f"{end_to_end.param_path.as_posix()}'",
+        ],
+    )
+
     env = os.environ.copy()
     env["ECOSCOPE_WORKFLOWS_RESULTS"] = tmp.as_posix()
-    out = subprocess.run(cmd, capture_output=True, text=True, env=env)
-    assert out.returncode == 0
+
+    proc = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        env=env,
+        shell=True,
+    )
+    returncode = proc.wait()
+    if returncode != 0:
+        assert proc.stderr is not None
+        raise ValueError(f"{cmd = } failed with:\n {proc.stderr.read()}")
+    assert returncode == 0
+    assert proc.stdout is not None
+    stdout = proc.stdout.read().strip()
     for assert_fn in end_to_end.assert_that_stdout:
-        assert assert_fn(out.stdout.strip())
+        assert assert_fn(stdout)