Cache CRDS_PATH reffile contents in CI workflows (#7333)

* Remove dev pytest from tox * Cache pip wheels * Refactor tox.ini into check- and test- groups * Cache CRDS_PATH in CI workflows * Cache pip wheels in setup-python action
spacetelescope · Nov 29, 2022 · 478ced4 · 478ced4
1 parent fa17e4c
commit 478ced4
Show file tree

Hide file tree

Showing 3 changed files with 142 additions and 151 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -11,6 +11,7 @@ on:
   schedule:
     # Weekly Monday 9AM build
     - cron: "0 9 * * 1"
+  workflow_dispatch:
 
 env:
   CRDS_SERVER_URL: https://jwst-crds.stsci.edu
@@ -19,82 +20,84 @@ env:
   CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
 
 jobs:
-  tox:
-    name: ${{ matrix.name }}
+  check:
+    name: ${{ matrix.toxenv }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: Python 3.10 Tests
-            python-version: '3.10'
-            os: ubuntu-latest
-            toxenv: py310
+        toxenv: [check-style, check-security, check-dependencies, check-build]
+        python-version: ['3.x']
+        os: [ubuntu-latest]
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: setup.cfg
+      - name: Run ${{ matrix.toxenv }}
+        run: |
+          pip install tox
+          tox -e ${{ matrix.toxenv }}
 
-          - name: Latest dependency versions w/coverage
+  test:
+    name: ${{ matrix.toxenv }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: test latest dependency versions
             os: ubuntu-latest
-            python-version: 3.9
-            toxenv: py39-cov
-
-          - name: Oldest dependency versions
+            python-version: "3.x"
+            toxenv: test-xdist-cov
+          - name: test oldest dependency versions
             os: ubuntu-latest
             python-version: 3.8
-            toxenv: py38-oldestdeps-cov
-
-          - name: SDP dependencies in requirements-sdp.txt
+            toxenv: test-oldestdeps-xdist-cov
+          - name: test SDP dependencies in requirements-sdp.txt
             os: ubuntu-latest
             python-version: 3.9
-            toxenv: sdpdeps
-
-          - name: Installed package with --pyargs
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: pyargs
-
-          - name: Verify install_requires in setup.py
+            toxenv: test-sdpdeps-xdist
+          - name: test installed package with --pyargs
             os: ubuntu-latest
-            python-version: 3.9
-            toxenv: verify-install-requires
-
-          - name: Build distribution
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: twine
-
-          - name: Code style check
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: style
-
-          - name: Security audit
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: security
-
-          # MacOS job is flaky on Github actions and fails routinely.
-          # Re-enable when things get more reliable.
-          # - name: macOS
-          #   os: macos-latest
-          #   python-version: 3.9
-          #   toxenv: py39
+            python-version: '3.10'
+            toxenv: test-pyargs-xdist
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: Install tox
+          cache: 'pip'
+          cache-dependency-path: setup.cfg
+      - name: Get CRDS context
+        id: crds-context
+        # Get default CRDS_CONTEXT without installing crds client
+        # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request
+        run: >
+          echo "pmap=$(
+          curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' https://jwst-crds.stsci.edu/json/ |
+          python -c "import sys, json; print(json.load(sys.stdin)['result'])"
+          )" >> $GITHUB_OUTPUT
+      - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}"
+      - name: Cache CRDS reference files
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-reffiles-${{ steps.crds-context.outputs.pmap }}
+      - name: Run ${{ matrix.toxenv }}
         run: |
           pip install tox
-
-      - name: Run tests
-        run: tox -e ${{ matrix.toxenv }}
-
+          tox -e ${{ matrix.toxenv }}
       - name: Upload coverage to codecov
         if: ${{ contains(matrix.toxenv,'-cov') }}
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml
@@ -8,49 +8,47 @@ on:
 
 env:
   CRDS_SERVER_URL: https://jwst-crds.stsci.edu
-  CRDS_PATH: ~/crds_cache
+  CRDS_PATH: $HOME/crds_cache
   CRDS_CLIENT_RETRY_COUNT: 3
   CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
 
 jobs:
-  tox:
-    name: ${{ matrix.name }}
+  test:
+    name: ${{ matrix.toxenv }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: Dev dependencies in requirements-dev.txt
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: devdeps
+        toxenv: [test-devdeps-xdist]
+        python-version: ['3.x']
+        os: [ubuntu-latest]
     steps:
-      - name: Install system packages
-        if: ${{ contains(matrix.toxenv,'docs') }}
-        run: |
-          sudo apt-get install graphviz texlive-latex-extra dvipng
-
-      - name: Checkout the code
+      - name: Checkout code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-
-      - name: Install tox
+          cache: 'pip'
+          cache-dependency-path: setup.cfg
+      - name: Get CRDS context
+        id: crds-context
+        # Get default CRDS_CONTEXT without installing crds client
+        # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request
+        run: >
+          echo "pmap=$(
+          curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' https://jwst-crds.stsci.edu/json/ |
+          python -c "import sys, json; print(json.load(sys.stdin)['result'])"
+          )" >> $GITHUB_OUTPUT
+      - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}"
+      - name: Cache CRDS reference files
+        uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-reffiles-${{ steps.crds-context.outputs.pmap }}
+      - name: Run ${{ matrix.toxenv }}
         run: |
           pip install tox
-
-      - name: Run tests
-        run: tox -e ${{ matrix.toxenv }}
-
-      - name: Upload coverage to codecov
-        if: ${{ contains(matrix.toxenv,'-cov') }}
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-          flags: unit
-          fail_ci_if_error: true
+          tox -e ${{ matrix.toxenv }}
diff --git a/tox.ini b/tox.ini
@@ -1,11 +1,10 @@
 [tox]
 envlist =
-    py{38,39,310}{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs}{,-cov}
-    style
-    security
+    check-{style,security,dependencies,build}
+    test{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs,-warnings,-regtests,-cov}
+    build-docs
 isolated_build = true
 
-[testenv]
 # tox environments are constructed with so-called 'factors' (or terms)
 # separated by hyphens, e.g. test-devdeps-cov. Lines below starting with factor:
 # will only take effect if that factor is included in the environment name. To
@@ -14,17 +13,51 @@ isolated_build = true
 #
 #     tox -l -v
 #
+
+[testenv:check-style]
+description = check code style, e.g. with flake8
+skip_install = true
+deps =
+    flake8
+commands =
+    flake8 . {posargs}
+
+[testenv:check-security]
+description = run bandit to check security compliance
+skip_install = true
+deps =
+    bandit>=1.7
+commands =
+    bandit -r -ll -x jwst/*test*,jwst/**/*test*,jwst/fits_generator jwst scripts
+
+[testenv:check-dependencies]
+description = verify that install_requires in setup.cfg has correct dependencies
+commands =
+    verify_install_requires
+
+[testenv:check-build]
+description = check build sdist/wheel and a strict twine check for metadata
+skip_install = true
+deps =
+    twine>=3.3
+    build
+commands =
+    python -m build .
+    twine check --strict dist/*
+
+[testenv]
 description =
     run tests
     devdeps: with the latest developer version of key dependencies
     oldestdeps: with the oldest supported version of key dependencies
     sdpdeps: with the recent STScI DMS release pinned dependencies
     pyargs: with --pyargs on installed package
-    cov: and test coverage
-
+    warnings: treating warnings as errors
+    regtests: with --bigdata and --slow flags
+    cov: with coverage
+    xdist: using parallel processing
 # The following indicates which extras_require from setup.cfg will be installed
 extras = test
-
 # Pass through the following environment variables which may be needed for the CI
 passenv =
     TOXENV
@@ -36,80 +69,37 @@ passenv =
     PASS_INVALID_VALUES
     VALIDATE_ON_ASSIGNMENT
     TEST_BIGDATA
-
-usedevelop = true
-
+usedevelop =
+    pyargs: false
+    !pyargs: true
 commands =
-    !cov: pytest -n auto {posargs}
-    cov: pytest -n auto --cov-report=xml --cov=. --cov-config=setup.cfg {posargs}
-
+    pip freeze
+    pytest \
+    cov: --cov=. --cov-config=setup.cfg --cov-report=xml \
+    warnings: -W error \
+    regtests: --bigdata --slow --basetemp={homedir}/test_outputs \
+    xdist: -n auto \
+    pyargs: {toxinidir}/docs --pyargs jwst \
+    {posargs}
 deps =
-    pytest-xdist
+    xdist: pytest-xdist
     devdeps: -rrequirements-dev.txt
     sdpdeps: -rrequirements-sdp.txt
-
-    py310: git+https://github.com/pytest-dev/pytest
-
 setenv =
     sdpdeps,regtests: CRDS_CONTEXT = jwst-edit
-
 commands_pre =
     python -m pip install --upgrade pip
     # Generate a requirements-min.txt file
     oldestdeps: minimum_deps
     # Force install everything in that file
     oldestdeps: pip install --ignore-installed -r requirements-min.txt
-
 # Don't treat positional arguments passed to tox as file system paths
-args_are_paths = false
-
-[testenv:pyargs]
-changedir = {homedir}
-usedevelop = false
-commands =
-    pyargs: pytest -n auto {toxinidir}/docs --pyargs {posargs:jwst}
-
-[testenv:regtests]
-description = run tests with --bigdata and --slow flags
-commands =
-    pytest -n auto --bigdata --slow --basetemp={homedir}/test_outputs {posargs}
-
-[testenv:twine]
-description = check that the package builds sdist/wheel and that twine uploads
-deps =
-    twine>=3.3
-    pep517
-usedevelop = false
-commands =
-    python -m pep517.check .
-    twine check --strict {distdir}/*
-
-[testenv:style]
-description = check code style, e.g. with flake8
-skip_install = true
-deps =
-    flake8
-commands =
-    flake8 {posargs}
-
-[testenv:verify-install-requires]
-description = verify that install_requires in setup.cfg is correct
-extras =
-commands =
-    verify_install_requires
-
-[testenv:warnings]
-commands =
-    pytest -W error {posargs}
-
-[testenv:security]
-description = run bandit to check security compliance
-deps =
-    bandit>=1.7
-commands =
-    bandit -r -ll -x jwst/*test*,jwst/**/*test*,jwst/fits_generator jwst scripts
+args_are_paths =
+    !pyargs: false
+changedir =
+    pyargs: {homedir}
 
-[testenv:docs]
+[testenv:build-docs]
 description = invoke sphinx-build to build the HTML docs
 extras = docs
 commands =