Skip to content
Backend Code Checks

Fix CI, static_check and db_dataset.yml (#5255) #10696

Sign in to view logs

Fix CI, static_check and db_dataset.yml (#5255)

Fix CI, static_check and db_dataset.yml (#5255) #10696

Workflow file for this run

.github/workflows/backend_checks.yml at fec1484
name: Backend Code Checks
on:
pull_request:
# This only ignores paths if there are _no_ commits on the branch that touch other files
paths-ignore:
- "**.md"
- "clients/**"
- ".vscode/**"
push:
branches:
- "main"
- "release-**"
env:
IMAGE: ethyca/fides:local
DEFAULT_PYTHON_VERSION: "3.10.13"
jobs:
###############
## Prechecks ##
###############
Collect-Tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Run Static Check
run: nox -s collect_tests
Build:
needs: Collect-Tests
strategy:
matrix:
# NOTE: These are the currently supported/tested Python Versions
python_version: ["3.9.18", "3.10.13"]
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v3
- name: Build container
uses: docker/build-push-action@v5
with:
builder: ${{ steps.buildx.outputs.name }}
context: .
build-args: PYTHON_VERSION=${{ matrix.python_version }}
target: prod
outputs: type=docker,dest=/tmp/python-${{ matrix.python_version }}.tar
push: false
tags: ${{ env.IMAGE }}
- name: Upload container
uses: actions/upload-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/python-${{ matrix.python_version }}.tar
retention-days: 1
###################
## Static Checks ##
###################
Static-Checks:
strategy:
matrix:
session_name:
[
'"isort(check)"',
'"black(check)"',
"mypy",
"pylint",
"xenon",
"check_install",
'"pytest(nox)"',
]
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Install Dev Requirements
run: pip install -r dev-requirements.txt
- name: Run Static Check
run: nox -s ${{ matrix.session_name }}
##################
## Performance ##
##################
Performance-Checks:
needs: Check-Container-Startup
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ env.DEFAULT_PYTHON_VERSION }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Rust/Cargo
run: curl -y https://sh.rustup.rs -sSf | sh
- name: Install Drill
run: cargo install drill
- name: Install Nox
run: pip install nox>=2022
- name: Run Performance Tests
run: nox -s performance_tests
#################
## Misc Checks ##
#################
Check-Container-Startup:
needs: Build
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ env.DEFAULT_PYTHON_VERSION }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Run test suite
run: nox -s check_container_startup
Misc-Tests:
needs: Check-Container-Startup
strategy:
matrix:
test_selection:
- "check_fides_annotations"
- "fides_db_scan"
- "docs_check"
- "minimal_config_startup"
runs-on: ubuntu-latest
timeout-minutes: 15
continue-on-error: true
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ env.DEFAULT_PYTHON_VERSION }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ env.DEFAULT_PYTHON_VERSION }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Run test suite
run: nox -s "${{ matrix.test_selection }}"
################
## Safe Tests ##
################
Safe-Tests:
needs: Check-Container-Startup
strategy:
matrix:
python_version: ["3.9.18", "3.10.13"]
test_selection:
- "ctl-not-external"
- "ops-unit-api"
- "ops-unit-non-api"
- "ops-integration"
- "lib"
runs-on: ubuntu-latest
timeout-minutes: 45
continue-on-error: true
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Set Up Python
uses: actions/setup-python@v5
with:
python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
cache: "pip"
- name: Install Nox
run: pip install nox>=2022
- name: Run test suite
run: nox -s "pytest(${{ matrix.test_selection }})"
- name: Upload coverage
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true
##################
## Unsafe Tests ##
##################
# NOTE: Matrixes aren't used here due to the danger of race conditions for external resources
Pytest-Ctl-External:
needs: Check-Container-Startup
strategy:
max-parallel: 1 # This prevents collisions in shared external resources
matrix:
python_version: ["3.9.18", "3.10.13"]
runs-on: ubuntu-latest
timeout-minutes: 20
# In PRs run with the "unsafe" label, or run on a "push" event to main
if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Install Nox
run: pip install nox>=2022
- name: Run external test suite
run: nox -s "pytest(ctl-external)"
env:
SNOWFLAKE_FIDESCTL_PASSWORD: ${{ secrets.SNOWFLAKE_FIDESCTL_PASSWORD }}
REDSHIFT_FIDESCTL_PASSWORD: ${{ secrets.REDSHIFT_FIDESCTL_PASSWORD }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CTL_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CTL_SECRET_ACCESS_KEY }}
OKTA_CLIENT_TOKEN: ${{ secrets.OKTA_FIDESCTL_CLIENT_TOKEN }}
AWS_DEFAULT_REGION: us-east-1
BIGQUERY_CONFIG: ${{ secrets.BIGQUERY_CONFIG }}
DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }}
DYNAMODB_ACCESS_KEY_ID: ${{ secrets.DYNAMODB_ACCESS_KEY_ID }}
DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }}
External-Datastores:
needs: Check-Container-Startup
strategy:
max-parallel: 1 # This prevents collisions in shared external resources
matrix:
python_version: ["3.9.18", "3.10.13"]
runs-on: ubuntu-latest
timeout-minutes: 20
# In PRs run with the "unsafe" label, or run on a "push" event to main
if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Checkout
uses: actions/checkout@v4
- name: Install Nox
run: pip install nox>=2022
- name: Integration Tests (External)
env:
BIGQUERY_DATASET: fidesopstest
BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }}
DYNAMODB_ACCESS_KEY_ID: ${{ secrets.DYNAMODB_ACCESS_KEY_ID }}
DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }}
DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }}
GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER }}
GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME }}
GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME }}
GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS }}
GOOGLE_CLOUD_SQL_POSTGRES_DB_IAM_USER: ${{ secrets.GOOGLE_CLOUD_SQL_POSTGRES_DB_IAM_USER }}
GOOGLE_CLOUD_SQL_POSTGRES_INSTANCE_CONNECTION_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_POSTGRES_INSTANCE_CONNECTION_NAME }}
GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_NAME }}
GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_SCHEMA_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_POSTGRES_DATABASE_SCHEMA_NAME }}
GOOGLE_CLOUD_SQL_POSTGRES_KEYFILE_CREDS: ${{ secrets.GOOGLE_CLOUD_SQL_POSTGRES_KEYFILE_CREDS }}
REDSHIFT_TEST_DATABASE: ${{ secrets.REDSHIFT_TEST_DATABASE }}
REDSHIFT_TEST_DB_SCHEMA: ${{ secrets.REDSHIFT_TEST_DB_SCHEMA }}
REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
REDSHIFT_TEST_PASSWORD: ${{ secrets.REDSHIFT_TEST_PASSWORD }}
REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER }}
SNOWFLAKE_TEST_DATABASE_NAME: ${{ secrets.SNOWFLAKE_TEST_DATABASE_NAME }}
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
SNOWFLAKE_TEST_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY }}
SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_TEST_PRIVATE_KEY_PASSPHRASE }}
SNOWFLAKE_TEST_SCHEMA_NAME: ${{ secrets.SNOWFLAKE_TEST_SCHEMA_NAME }}
SNOWFLAKE_TEST_USER_LOGIN_NAME: ${{ secrets.SNOWFLAKE_TEST_USER_LOGIN_NAME }}
SNOWFLAKE_TEST_WAREHOUSE_NAME: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE_NAME }}
run: nox -s "pytest(ops-external-datastores)"
External-SaaS-Connectors:
needs: Check-Container-Startup
runs-on: ubuntu-latest
timeout-minutes: 30
# In PRs run with the "unsafe" label, or run on a "push" event to main
if: contains(github.event.pull_request.labels.*.name, 'run unsafe ci checks') || github.event_name == 'push'
permissions:
contents: read
id-token: write
strategy:
max-parallel: 1 # This prevents collisions in shared external resources
matrix:
python_version: ["3.9.18", "3.10.13"]
steps:
- name: Download container
uses: actions/download-artifact@v4
with:
name: python-${{ matrix.python_version }}
path: /tmp/
- name: Load image
run: docker load --input /tmp/python-${{ matrix.python_version }}.tar
- name: Install Nox
run: pip install nox>=2022
- name: Checkout
uses: actions/checkout@v4
- name: Get Vault Token
uses: hashicorp/[email protected]
with:
url: ${{ secrets.VAULT_ADDR }}
namespace: ${{ secrets.VAULT_NAMESPACE }}
method: jwt
role: ${{ secrets.VAULT_ROLE }}
exportToken: True
- name: SaaS Connector Tests
env:
VAULT_ADDR: ${{ secrets.VAULT_ADDR }}
VAULT_NAMESPACE: ${{ secrets.VAULT_NAMESPACE }}
run: nox -s "pytest(ops-saas)"