use GitHub Actions to cache environments and use dependent CI workflo…

…w structure (#97)

* use GitHub Actions to cache environments and use dependent CI workflow structure

* use editable install in tests

* coverage dependent on tests

* add `CRDS_PATH` to local variables

* run `jwst.datamodels` tests in parallel

* add more variables from JWST tests

* attempt `crds sync --all` beforehand

* use `~` in user path

* hardcode user dir

* cache CRDS mappings used in `jwst.datamodels` tests

* build a new cache on version change of CRDS context

.github/workflows/cancel_workflows.yml

@@ -0,0 +1,17 @@
+name: Cancel duplicate workflows
+
+on:
+  workflow_run:
+    workflows: ["CI"]
+    types:
+      - requested
+
+# Note: This has to be in workflow_run so it works for PRs from forks.
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Cancel previous runs
+      uses: styfle/cancel-workflow-action@3d86a7cc43670094ac248017207be0295edbc31d  # 0.8.0
+      with:
+        workflow_id: ${{ github.event.workflow.id }}

.github/workflows/ci.yml

@@ -14,64 +14,118 @@ on:
     - cron: '0 9 * * 1'
 
 jobs:
-  tox:
-    name: ${{ matrix.name }}
-    runs-on: ${{ matrix.runs-on }}
+  style:
+    name: Code style checks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: style-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install pyproject-flake8
+      - run: pflake8 --count src
+  audit:
+    name: Bandit security audit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: audit-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install bandit
+      - run: bandit -r -ll src
+  test:
+    name: test
+    needs: [ style, audit ]
+    runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - name: Python 3.9
-            runs-on: ubuntu-latest
-            python-version: 3.9
-            toxenv: py39
-
-          - name: Coverage
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: coverage
-
-          - name: Code style checks
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: style
-
-          - name: Bandit security audit
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: bandit
-
-          - name: macOS
-            runs-on: macos-latest
-            python-version: 3.8
-            toxenv: py38
-
-          - name: jwst.datamodels tests
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: jwst
-
-          - name: Build documentation
-            runs-on: ubuntu-latest
-            python-version: 3.8
-            toxenv: build-docs
-
+        python: [ 3.8, 3.9, '3.10' ]
+        os: [ ubuntu-latest, macos-latest ]
     steps:
-      - name: Install extra system packages
-        if: ${{ contains(matrix.toxenv,'docs') }}
-        run: |
-          sudo apt update -y
-          sudo apt-get install graphviz texlive-latex-extra dvipng
       - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
         with:
-          fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+          python-version: ${{ matrix.python }}
+      - uses: actions/cache@v3
         with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install tox
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-      - name: Run tox
-        run: tox -e ${{ matrix.toxenv }}
+          path: ${{ env.pythonLocation }}
+          key: test-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install -e ".[test]" pytest-xdist
+      - run: pip freeze
+      - run: pytest -n auto
+  test_jwst:
+    name: jwst.datamodels tests
+    needs: [ style, audit ]
+    runs-on: ubuntu-latest
+    env:
+      CRDS_SERVER_URL: https://jwst-crds.stsci.edu
+      CRDS_PATH: /home/runner/work/stdatamodels/crds_cache
+      CRDS_CLIENT_RETRY_COUNT: 3
+      CRDS_CLIENT_RETRY_DELAY_SECONDS: 20
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: test-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install -e ".[test]" pytest-xdist
+      - run: pip install "jwst[test] @ git+https://github.com/spacetelescope/jwst.git"
+      - run: export CRDS_VERSION=$(crds list --contexts jwst_0865.pmap --version)
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.CRDS_PATH }}
+          key: crds-jwst-datamodels-${{ env.CRDS_VERSION }}
+      - run: crds sync --contexts jwst_0865.pmap
+      - run: pip freeze
+      - run: pytest -n auto --pyargs jwst.datamodels
+  test_with_coverage:
+    name: Coverage
+    needs: [ test, test_jwst ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: test-coverage-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install -e ".[test]" pytest-xdist pytest-cov
+      - run: pip install "asdf @ git+https://github.com/asdf-format/asdf.git"
+      - run: pip freeze
+      - run: pytest -n auto --cov-report=xml --cov=src/stdatamodels
+      - run: coverage report -m
+      - uses: codecov/codecov-action@v2
+        with:
+          file: ./coverage.xml
+  build-docs:
+    name: Build documentation
+    needs: [ test, test_jwst ]
+    runs-on: ubuntu-latest
+    steps:
+      - run: sudo apt-get install graphviz texlive-latex-extra dvipng
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/cache@v3
+        with:
+          path: ${{ env.pythonLocation }}
+          key: build-docs-${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('**/pyproject.toml', '**/setup.*') }}
+      - run: pip install -e ".[docs]"
+      - run: sphinx-build -W docs/source build/docs
+