separate tox testenvs to cache CRDS reference files retrieved by init…

…ial test
spacetelescope · Oct 28, 2022 · 98b5be4 · 98b5be4
1 parent b262b28
commit 98b5be4
Show file tree

Hide file tree

Showing 2 changed files with 139 additions and 95 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,86 +19,139 @@ env:
   CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
 
 jobs:
-  tox:
+  checks:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: false
       matrix:
         include:
-          - name: Python 3.10 Tests
-            python-version: '3.10'
-            os: ubuntu-latest
-            toxenv: py310
-
-          - name: Latest dependency versions w/coverage
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: py39-cov
-
-          - name: Oldest dependency versions
-            os: ubuntu-latest
-            python-version: 3.8
-            toxenv: py38-oldestdeps-cov
-
-          - name: SDP dependencies in requirements-sdp.txt
+          - name: Code style check
             os: ubuntu-latest
             python-version: 3.9
-            toxenv: sdpdeps
-
-          - name: Installed package with --pyargs
+            toxenv: style
+          - name: Security audit
             os: ubuntu-latest
             python-version: 3.9
-            toxenv: pyargs
-
+            toxenv: security
           - name: Verify install_requires in setup.py
             os: ubuntu-latest
             python-version: 3.9
             toxenv: verify-install-requires
-
-          - name: Build distribution
+  quick_test:
+    name: test Python ${{ matrix.python-version }}, tox testenv ${{ matrix.toxenv }}, ${{ matrix.os }}
+    needs: [ checks ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - python-version: '3.10'
             os: ubuntu-latest
+            toxenv: py310
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-reference-files-${{ hashFiles('${{ env.CRDS_PATH }}/reference') }}
+      - run: tox -e ${{ matrix.toxenv }}
+  test:
+    name: test (Python ${{ matrix.python-version }}, tox testenv ${{ matrix.toxenv }}, ${{ matrix.os }})
+    needs: [ quick_test ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-latest
             python-version: 3.9
-            toxenv: twine
-
-          - name: Code style check
+            toxenv: py39
+          - os: ubuntu-latest
+            python-version: 3.9
+            toxenv: pyargs
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-reference-files-${{ hashFiles('${{ env.CRDS_PATH }}/reference') }}
+      - run: tox -e ${{ matrix.toxenv }}
+  test_dependencies:
+    name: ${{ matrix.name }}
+    needs: [ test ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - name: Latest dependency versions w/coverage
             os: ubuntu-latest
             python-version: 3.9
-            toxenv: style
-
-          - name: Security audit
+            toxenv: py39-cov
+          - name: Oldest dependency versions w/coverage
+            os: ubuntu-latest
+            python-version: 3.8
+            toxenv: py38-oldestdeps-cov
+          - name: SDP dependencies in requirements-sdp.txt
             os: ubuntu-latest
             python-version: 3.9
-            toxenv: security
-
-          # MacOS job is flaky on Github actions and fails routinely.
-          # Re-enable when things get more reliable.
-          # - name: macOS
-          #   os: macos-latest
-          #   python-version: 3.9
-          #   toxenv: py39
+            toxenv: sdpdeps
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
-          fetch-depth: 0
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-reference-files-${{ hashFiles('${{ env.CRDS_PATH }}/reference') }}
+      - run: tox -e ${{ matrix.toxenv }}
+  test_with_coverage:
+    name: test with coverage (Python ${{ matrix.python-version }}, tox testenv ${{ matrix.toxenv }}, ${{ matrix.os }})
+    needs: [ test ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            python-version: 3.10
+            toxenv: py310-cov
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: Install tox
-        run: |
-          pip install tox
-
-      - name: Run tests
-        run: tox -e ${{ matrix.toxenv }}
-
-      - name: Upload coverage to codecov
-        if: ${{ contains(matrix.toxenv,'-cov') }}
+      - run: pip install tox
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-reference-files-${{ hashFiles('${{ env.CRDS_PATH }}/reference') }}
+      - run: tox -e ${{ matrix.toxenv }}
+      - if: ${{ contains(matrix.toxenv,'-cov') }}
         uses: codecov/codecov-action@v3
         with:
           file: ./coverage.xml
           flags: unit
           fail_ci_if_error: true
+  build:
+    name: build distribution (Python ${{ matrix.python-version }}, tox testenv ${{ matrix.toxenv }}, ${{ matrix.os }})
+    needs: [ quick_test ]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ 'ubuntu-latest', 'macos-latest' ]
+        python-version: [ '3.9', '3.10', '3.11' ]
+        toxenv: [ 'twine' ]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - run: tox -e ${{ matrix.toxenv }}
diff --git a/tox.ini b/tox.ini
@@ -1,8 +1,8 @@
 [tox]
 envlist =
-    py{38,39,310}{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs}{,-cov}
     style
     security
+    py{38,39,310,311}{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs}{,-cov}
 isolated_build = true
 
 [testenv]
@@ -20,11 +20,9 @@ description =
     oldestdeps: with the oldest supported version of key dependencies
     sdpdeps: with the recent STScI DMS release pinned dependencies
     pyargs: with --pyargs on installed package
-    cov: and test coverage
-
+    cov: with coverage
 # The following indicates which extras_require from setup.cfg will be installed
 extras = test
-
 # Pass through the following environment variables which may be needed for the CI
 passenv =
     TOXENV
@@ -36,39 +34,57 @@ passenv =
     PASS_INVALID_VALUES
     VALIDATE_ON_ASSIGNMENT
     TEST_BIGDATA
-
 usedevelop = true
-
 commands =
+    pip freeze
     !cov: pytest -n auto {posargs}
-    cov: pytest -n auto --cov-report=xml --cov=. --cov-config=setup.cfg {posargs}
-
+    cov: pytest -n auto --cov=. --cov-config=setup.cfg --cov-report=term-missing --cov-report=xml {posargs}
 deps =
     pytest-xdist
     devdeps: -rrequirements-dev.txt
     sdpdeps: -rrequirements-sdp.txt
-
-    py310: git+https://github.com/pytest-dev/pytest
-
 setenv =
     sdpdeps,regtests: CRDS_CONTEXT = jwst-edit
-
 commands_pre =
     python -m pip install --upgrade pip
-    # Generate a requirements-min.txt file
+# Generate a requirements-min.txt file
     oldestdeps: minimum_deps
-    # Force install everything in that file
+# Force install everything in that file
     oldestdeps: pip install --ignore-installed -r requirements-min.txt
-
 # Don't treat positional arguments passed to tox as file system paths
 args_are_paths = false
 
+[testenv:style]
+description = check code style, e.g. with flake8
+skip_install = true
+deps =
+    flake8 .
+commands =
+    flake8 {posargs}
+
+[testenv:security]
+description = run bandit to check security compliance
+deps =
+    bandit>=1.7
+commands =
+    bandit -r -ll -x jwst/*test*,jwst/**/*test*,jwst/fits_generator jwst scripts
+
+[testenv:verify-install-requires]
+description = verify that install_requires in setup.cfg is correct
+extras =
+commands =
+    verify_install_requires
+
 [testenv:pyargs]
 changedir = {homedir}
 usedevelop = false
 commands =
     pyargs: pytest -n auto {toxinidir}/docs --pyargs {posargs:jwst}
 
+[testenv:warnings]
+commands =
+    pytest -W error {posargs}
+
 [testenv:regtests]
 description = run tests with --bigdata and --slow flags
 commands =
@@ -84,31 +100,6 @@ commands =
     python -m pep517.check .
     twine check --strict {distdir}/*
 
-[testenv:style]
-description = check code style, e.g. with flake8
-skip_install = true
-deps =
-    flake8
-commands =
-    flake8 {posargs}
-
-[testenv:verify-install-requires]
-description = verify that install_requires in setup.cfg is correct
-extras =
-commands =
-    verify_install_requires
-
-[testenv:warnings]
-commands =
-    pytest -W error {posargs}
-
-[testenv:security]
-description = run bandit to check security compliance
-deps =
-    bandit>=1.7
-commands =
-    bandit -r -ll -x jwst/*test*,jwst/**/*test*,jwst/fits_generator jwst scripts
-
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs
 extras = docs