From 15a019fe37ff465b0483725957d93a98c1c4bab0 Mon Sep 17 00:00:00 2001 From: James Davies Date: Wed, 2 Nov 2022 23:07:42 +0100 Subject: [PATCH 1/5] Remove dev pytest from tox --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8f6ba90e6a..2f5a5e864a 100644 --- a/tox.ini +++ b/tox.ini @@ -48,8 +48,6 @@ deps = devdeps: -rrequirements-dev.txt sdpdeps: -rrequirements-sdp.txt - py310: git+https://github.com/pytest-dev/pytest - setenv = sdpdeps,regtests: CRDS_CONTEXT = jwst-edit From 23f420b6ace80be0cef2dae87e7a84020945abac Mon Sep 17 00:00:00 2001 From: James Davies Date: Wed, 2 Nov 2022 23:21:40 +0100 Subject: [PATCH 2/5] Cache pip wheels --- .github/workflows/ci.yml | 43 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d934a66f47..ca1cb58eee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ on: schedule: # Weekly Monday 9AM build - cron: "0 9 * * 1" + workflow_dispatch: env: CRDS_SERVER_URL: https://jwst-crds.stsci.edu @@ -51,25 +52,25 @@ jobs: python-version: 3.9 toxenv: pyargs - - name: Verify install_requires in setup.py - os: ubuntu-latest - python-version: 3.9 - toxenv: verify-install-requires + # - name: Verify install_requires in setup.py + # os: ubuntu-latest + # python-version: 3.9 + # toxenv: verify-install-requires - - name: Build distribution - os: ubuntu-latest - python-version: 3.9 - toxenv: twine + # - name: Build distribution + # os: ubuntu-latest + # python-version: 3.9 + # toxenv: twine - - name: Code style check - os: ubuntu-latest - python-version: 3.9 - toxenv: style + # - name: Code style check + # os: ubuntu-latest + # python-version: 3.9 + # toxenv: style - - name: Security audit - os: ubuntu-latest - python-version: 3.9 - toxenv: security + # - name: Security audit + # os: ubuntu-latest + # python-version: 3.9 + # toxenv: security # MacOS job is flaky on Github actions and fails routinely. # Re-enable when things get more reliable. @@ -82,19 +83,19 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 + id: setup with: python-version: ${{ matrix.python-version }} - + cache: 'pip' + cache-dependency-path: setup.cfg + - run: echo '${{ steps.setup.outputs.cache-hit }}' - name: Install tox run: | pip install tox - - name: Run tests run: tox -e ${{ matrix.toxenv }} - - name: Upload coverage to codecov if: ${{ contains(matrix.toxenv,'-cov') }} uses: codecov/codecov-action@v3 From 79f51514639897d25b0816467812a3aea5245b3e Mon Sep 17 00:00:00 2001 From: James Davies Date: Fri, 4 Nov 2022 10:31:59 +0100 Subject: [PATCH 3/5] Refactor tox.ini into check- and test- groups --- tox.ini | 122 ++++++++++++++++++++++++++------------------------------ 1 file changed, 57 insertions(+), 65 deletions(-) diff --git a/tox.ini b/tox.ini index 2f5a5e864a..6925c79ca1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,10 @@ [tox] envlist = - py{38,39,310}{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs}{,-cov} - style - security + check-{style,security,dependencies,build} + test{,-oldestdeps,-devdeps,-sdpdeps}{,-pyargs,-warnings,-regtests,-cov} + build-docs isolated_build = true -[testenv] # tox environments are constructed with so-called 'factors' (or terms) # separated by hyphens, e.g. test-devdeps-cov. Lines below starting with factor: # will only take effect if that factor is included in the environment name. To @@ -14,17 +13,51 @@ isolated_build = true # # tox -l -v # + +[testenv:check-style] +description = check code style, e.g. with flake8 +skip_install = true +deps = + flake8 +commands = + flake8 . {posargs} + +[testenv:check-security] +description = run bandit to check security compliance +skip_install = true +deps = + bandit>=1.7 +commands = + bandit -r -ll -x jwst/*test*,jwst/**/*test*,jwst/fits_generator jwst scripts + +[testenv:check-dependencies] +description = verify that install_requires in setup.cfg has correct dependencies +commands = + verify_install_requires + +[testenv:check-build] +description = check build sdist/wheel and a strict twine check for metadata +skip_install = true +deps = + twine>=3.3 + build +commands = + python -m build . + twine check --strict dist/* + +[testenv] description = run tests devdeps: with the latest developer version of key dependencies oldestdeps: with the oldest supported version of key dependencies sdpdeps: with the recent STScI DMS release pinned dependencies pyargs: with --pyargs on installed package - cov: and test coverage - + warnings: treating warnings as errors + regtests: with --bigdata and --slow flags + cov: with coverage + xdist: using parallel processing # The following indicates which extras_require from setup.cfg will be installed extras = test - # Pass through the following environment variables which may be needed for the CI passenv = TOXENV @@ -36,78 +69,37 @@ passenv = PASS_INVALID_VALUES VALIDATE_ON_ASSIGNMENT TEST_BIGDATA - -usedevelop = true - +usedevelop = + pyargs: false + !pyargs: true commands = - !cov: pytest -n auto {posargs} - cov: pytest -n auto --cov-report=xml --cov=. --cov-config=setup.cfg {posargs} - + pip freeze + pytest \ + cov: --cov=. --cov-config=setup.cfg --cov-report=xml \ + warnings: -W error \ + regtests: --bigdata --slow --basetemp={homedir}/test_outputs \ + xdist: -n auto \ + pyargs: {toxinidir}/docs --pyargs jwst \ + {posargs} deps = - pytest-xdist + xdist: pytest-xdist devdeps: -rrequirements-dev.txt sdpdeps: -rrequirements-sdp.txt - setenv = sdpdeps,regtests: CRDS_CONTEXT = jwst-edit - commands_pre = python -m pip install --upgrade pip # Generate a requirements-min.txt file oldestdeps: minimum_deps # Force install everything in that file oldestdeps: pip install --ignore-installed -r requirements-min.txt - # Don't treat positional arguments passed to tox as file system paths -args_are_paths = false - -[testenv:pyargs] -changedir = {homedir} -usedevelop = false -commands = - pyargs: pytest -n auto {toxinidir}/docs --pyargs {posargs:jwst} - -[testenv:regtests] -description = run tests with --bigdata and --slow flags -commands = - pytest -n auto --bigdata --slow --basetemp={homedir}/test_outputs {posargs} - -[testenv:twine] -description = check that the package builds sdist/wheel and that twine uploads -deps = - twine>=3.3 - pep517 -usedevelop = false -commands = - python -m pep517.check . - twine check --strict {distdir}/* - -[testenv:style] -description = check code style, e.g. with flake8 -skip_install = true -deps = - flake8 -commands = - flake8 {posargs} - -[testenv:verify-install-requires] -description = verify that install_requires in setup.cfg is correct -extras = -commands = - verify_install_requires - -[testenv:warnings] -commands = - pytest -W error {posargs} - -[testenv:security] -description = run bandit to check security compliance -deps = - bandit>=1.7 -commands = - bandit -r -ll -x jwst/*test*,jwst/**/*test*,jwst/fits_generator jwst scripts +args_are_paths = + !pyargs: false +changedir = + pyargs: {homedir} -[testenv:docs] +[testenv:build-docs] description = invoke sphinx-build to build the HTML docs extras = docs commands = From 6b512db6bff4e3e66f2ee8b420381632bd5c6528 Mon Sep 17 00:00:00 2001 From: James Davies Date: Fri, 4 Nov 2022 10:32:31 +0100 Subject: [PATCH 4/5] Cache CRDS_PATH in CI workflows --- .github/workflows/ci.yml | 104 +++++++++++++++++----------------- .github/workflows/ci_cron.yml | 56 +++++++++--------- 2 files changed, 79 insertions(+), 81 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca1cb58eee..6b3c5c8120 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,64 +20,52 @@ env: CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 jobs: - tox: - name: ${{ matrix.name }} + check: + name: ${{ matrix.toxenv }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - name: Python 3.10 Tests - python-version: '3.10' - os: ubuntu-latest - toxenv: py310 + toxenv: [check-style, check-security, check-dependencies, check-build] + python-version: ['3.x'] + os: [ubuntu-latest] + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Run ${{ matrix.toxenv }} + run: | + pip install tox + tox -e ${{ matrix.toxenv }} - - name: Latest dependency versions w/coverage + test: + name: ${{ matrix.toxenv }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: test latest dependency versions os: ubuntu-latest - python-version: 3.9 - toxenv: py39-cov - - - name: Oldest dependency versions + python-version: "3.x" + toxenv: test-xdist-cov + - name: test oldest dependency versions os: ubuntu-latest python-version: 3.8 - toxenv: py38-oldestdeps-cov - - - name: SDP dependencies in requirements-sdp.txt + toxenv: test-oldestdeps-xdist-cov + - name: test SDP dependencies in requirements-sdp.txt os: ubuntu-latest python-version: 3.9 - toxenv: sdpdeps - - - name: Installed package with --pyargs + toxenv: test-sdpdeps-xdist + - name: test installed package with --pyargs os: ubuntu-latest - python-version: 3.9 - toxenv: pyargs - - # - name: Verify install_requires in setup.py - # os: ubuntu-latest - # python-version: 3.9 - # toxenv: verify-install-requires - - # - name: Build distribution - # os: ubuntu-latest - # python-version: 3.9 - # toxenv: twine - - # - name: Code style check - # os: ubuntu-latest - # python-version: 3.9 - # toxenv: style - - # - name: Security audit - # os: ubuntu-latest - # python-version: 3.9 - # toxenv: security - - # MacOS job is flaky on Github actions and fails routinely. - # Re-enable when things get more reliable. - # - name: macOS - # os: macos-latest - # python-version: 3.9 - # toxenv: py39 + python-version: '3.10' + toxenv: test-pyargs-xdist steps: - name: Checkout code uses: actions/checkout@v3 @@ -85,17 +73,29 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 - id: setup with: python-version: ${{ matrix.python-version }} cache: 'pip' cache-dependency-path: setup.cfg - - run: echo '${{ steps.setup.outputs.cache-hit }}' - - name: Install tox + - name: Get CRDS context + id: crds-context + # Get default CRDS_CONTEXT without installing crds client + # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request + run: > + echo "pmap=$( + curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' https://jwst-crds.stsci.edu/json/ | + python -c "import sys, json; print(json.load(sys.stdin)['result'])" + )" >> $GITHUB_OUTPUT + - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}" + - name: Cache CRDS reference files + uses: actions/cache@v3 + with: + path: ${{ env.CRDS_PATH }} + key: crds-reffiles-${{ steps.crds-context.outputs.pmap }} + - name: Run ${{ matrix.toxenv }} run: | pip install tox - - name: Run tests - run: tox -e ${{ matrix.toxenv }} + tox -e ${{ matrix.toxenv }} - name: Upload coverage to codecov if: ${{ contains(matrix.toxenv,'-cov') }} uses: codecov/codecov-action@v3 diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 1bde0ff2ee..4a08833a34 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -8,49 +8,47 @@ on: env: CRDS_SERVER_URL: https://jwst-crds.stsci.edu - CRDS_PATH: ~/crds_cache + CRDS_PATH: $HOME/crds_cache CRDS_CLIENT_RETRY_COUNT: 3 CRDS_CLIENT_RETRY_DELAY_SECONDS: 20 jobs: - tox: - name: ${{ matrix.name }} + test: + name: ${{ matrix.toxenv }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - include: - - name: Dev dependencies in requirements-dev.txt - os: ubuntu-latest - python-version: 3.9 - toxenv: devdeps + toxenv: [test-devdeps-xdist] + python-version: ['3.x'] + os: [ubuntu-latest] steps: - - name: Install system packages - if: ${{ contains(matrix.toxenv,'docs') }} - run: | - sudo apt-get install graphviz texlive-latex-extra dvipng - - - name: Checkout the code + - name: Checkout code uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - - name: Install tox + cache: 'pip' + cache-dependency-path: setup.cfg + - name: Get CRDS context + id: crds-context + # Get default CRDS_CONTEXT without installing crds client + # See https://hst-crds.stsci.edu/static/users_guide/web_services.html#generic-request + run: > + echo "pmap=$( + curl -s -X POST -d '{"jsonrpc": "1.0", "method": "get_default_context", "params": ["jwst"], "id": 1}' https://jwst-crds.stsci.edu/json/ | + python -c "import sys, json; print(json.load(sys.stdin)['result'])" + )" >> $GITHUB_OUTPUT + - run: echo "CRDS CONTEXT ${{ steps.crds-context.outputs.pmap }}" + - name: Cache CRDS reference files + uses: actions/cache@v3 + with: + path: ${{ env.CRDS_PATH }} + key: crds-reffiles-${{ steps.crds-context.outputs.pmap }} + - name: Run ${{ matrix.toxenv }} run: | pip install tox - - - name: Run tests - run: tox -e ${{ matrix.toxenv }} - - - name: Upload coverage to codecov - if: ${{ contains(matrix.toxenv,'-cov') }} - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unit - fail_ci_if_error: true + tox -e ${{ matrix.toxenv }} From 4690040ecc54577a895fc11263e949bd06b7e12d Mon Sep 17 00:00:00 2001 From: James Davies Date: Tue, 8 Nov 2022 12:11:28 +0100 Subject: [PATCH 5/5] Cache pip wheels in setup-python action --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b3c5c8120..0abe0b68ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + cache: 'pip' + cache-dependency-path: setup.cfg - name: Run ${{ matrix.toxenv }} run: | pip install tox