Skip to content

fix: retry schema-migration sfn on Batch.AWSBatchException #20554

fix: retry schema-migration sfn on Batch.AWSBatchException

fix: retry schema-migration sfn on Batch.AWSBatchException #20554

Workflow file for this run

name: Push Tests
on:
pull_request:
branches:
- "*"
push:
branches:
- "main"
workflow_call:
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
DEPLOYMENT_STAGE: test
# Force using BuildKit instead of normal Docker, required so that metadata
# is written/read to allow us to use layers of previous builds as cache.
DOCKER_BUILDKIT: 1
COMPOSE_DOCKER_CLI_BUILD: 1
DOCKER_REPO: ${{ secrets.ECR_REPO }}/
permissions:
id-token: write
contents: read
jobs:
lint:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- uses: actions/setup-python@v5
- name: check backend
uses: pre-commit/[email protected]
- uses: actions/setup-node@v4
with:
node-version-file: "frontend/.nvmrc"
cache: "npm"
cache-dependency-path: "frontend/package-lock.json"
- name: check frontend
run: |
cd frontend
npm ci
cp src/configs/local.js src/configs/configs.js
npm run lint
- uses: 8398a7/action-slack@v3
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
backend-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-backend
- name: Run unit tests
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-backend
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
backend-wmg-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-wmg-backend
- name: Run unit tests
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-wmg-backend
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
backend-de-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-de-backend
- name: Run unit tests
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-de-backend
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
processing-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile*
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-processing
- name: Run tests in docker-compose
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-processing
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
wmg-processing-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile*
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-wmg-processing
- name: Run tests in docker-compose
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-wmg-processing
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
cellguide-pipeline-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile*
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-cellguide-pipeline
- name: Run tests in docker-compose
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-cellguide-pipeline
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
cxg-admin-unit-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-backend
- name: Run unit tests
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-unit-test-cxg-admin
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
backend-integration-test:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Get changed files
id: changed-files
uses: tj-actions/[email protected]
with:
files: |
Dockerfile*
**/Dockerfile
requirements*.txt
**/requirements*.txt
- name: Check if containers need to be rebuilt
if: steps.changed-files.outputs.any_changed == 'true'
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-rebuild-backend
- name: Run tests against postgres instance
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make local-integration-test-backend
- name: Upload coverage results as an artifact
uses: actions/upload-artifact@v3
with:
name: coverage
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/.coverage*
retention-days: 3
- name: Upload Allure results as an artifact
uses: actions/upload-artifact@v3
with:
name: allure-results
path: /home/runner/work/single-cell-data-portal/single-cell-data-portal/allure-results
retention-days: 20
- uses: 8398a7/[email protected]
with:
status: ${{ job.status }}
fields: repo,commit,author,eventName,workflow,job,mention
mention: "here"
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
if: failure() && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/prod')
submit-codecoverage:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
needs:
- backend-unit-test
- backend-integration-test
- backend-wmg-unit-test
- wmg-processing-unit-test
- processing-unit-test
- cellguide-pipeline-unit-test
runs-on: ubuntu-22.04
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
aws-region: us-west-2
role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
role-duration-seconds: 1800
- name: Login to ECR
uses: docker/login-action@v3
with:
registry: ${{ secrets.ECR_REPO }}
- uses: actions/checkout@v3
with:
fetch-depth: 2
- uses: actions/download-artifact@v3
with:
name: coverage
path: .
- name: coverage report
run: |
echo "DOCKER_REPO=${DOCKER_REPO}" > .env.ecr
make coverage/report-xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
token: ${{ secrets.CODECOV_TOKEN }}
env_vars: OS,PYTHON
files: ./coverage.xml
flags: unittests
name: codecov-umbrella
publish-test-report:
if: ${{ ! startsWith(github.head_ref, 'renovate/') }}
needs:
- backend-unit-test
- backend-wmg-unit-test
- backend-integration-test
- wmg-processing-unit-test
- processing-unit-test
- cellguide-pipeline-unit-test
runs-on: ubuntu-22.04
permissions:
contents: write
steps:
- uses: actions/download-artifact@v3
with:
name: allure-results
path: allure-results
# Checkout gh-pages branch and get Allure result history
- name: Get Allure history
uses: actions/checkout@v3
if: always()
continue-on-error: true
with:
ref: gh-pages
path: gh-pages
fetch-depth: 2
# Create test results history as an Allure report
- name: Create test results history as an Allure report
uses: simple-elf/allure-report-action@master
if: always()
id: allure-report
with:
allure_results: allure-results
gh_pages: gh-pages
allure_report: allure-report
allure_history: allure-history
# Deploy Allure report to github pages
- name: Deploy report to Github Pages
if: always()
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_branch: gh-pages
publish_dir: allure-history
# Output the link to the github page for this current run
- name: Post the link to the report in summary
run: "echo 'Test Results Dashboard (Run #${{ github.run_number }}): https://chanzuckerberg.github.io/single-cell-data-portal/${{ github.run_number }}' >> $GITHUB_STEP_SUMMARY"