Skip to content

Run some example in Kubernetes execution mode in CI #3170

Run some example in Kubernetes execution mode in CI

Run some example in Kubernetes execution mode in CI #3170

Workflow file for this run

#name: test
#
#on:
# push: # Run on pushes to the default branch
# branches: [main]
# pull_request_target: # Also run on pull requests originated from forks
# branches: [main]
#
#concurrency:
# group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
# cancel-in-progress: true
#
#jobs:
# Authorize:
# environment: ${{ github.event_name == 'pull_request_target' &&
# github.event.pull_request.head.repo.full_name != github.repository &&
# 'external' || 'internal' }}
# runs-on: ubuntu-latest
# steps:
# - run: true
#
# Type-Check:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
#
# - uses: actions/setup-python@v3
# with:
# python-version: "3.9"
# architecture: "x64"
#
# - run: pip3 install hatch
# - run: hatch run tests.py3.9-2.7:type-check
#
# Run-Unit-Tests:
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
# airflow-version: ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"]
# exclude:
# - python-version: "3.11"
# airflow-version: "2.4"
# - python-version: "3.11"
# airflow-version: "2.5"
# # Apache Airflow versions prior to 2.9.0 have not been tested with Python 3.12.
# # Official support for Python 3.12 and the corresponding constraints.txt are available only for Apache Airflow >= 2.9.0.
# # See: https://github.com/apache/airflow/tree/2.9.0?tab=readme-ov-file#requirements
# # See: https://github.com/apache/airflow/tree/2.8.4?tab=readme-ov-file#requirements
# - python-version: "3.12"
# airflow-version: "2.4"
# - python-version: "3.12"
# airflow-version: "2.5"
# - python-version: "3.12"
# airflow-version: "2.6"
# - python-version: "3.12"
# airflow-version: "2.7"
# - python-version: "3.12"
# airflow-version: "2.8"
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
#
# - uses: actions/cache@v3
# with:
# path: |
# ~/.cache/pip
# .local/share/hatch/
# key: unit-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
#
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
#
# - name: Install packages and dependencies
# run: |
# python -m pip install uv
# uv pip install --system hatch
# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
#
# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
# run: |
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-cov
#
# - name: Upload coverage to Github
# uses: actions/upload-artifact@v2
# with:
# name: coverage-unit-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
# path: .coverage
#
# Run-Integration-Tests:
# needs: Authorize
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.8", "3.9", "3.10", "3.11"]
# airflow-version: ["2.4", "2.5", "2.6", "2.7", "2.8", "2.9"]
# exclude:
# - python-version: "3.11"
# airflow-version: "2.4"
# - python-version: "3.11"
# airflow-version: "2.5"
# services:
# postgres:
# image: postgres
# env:
# POSTGRES_PASSWORD: postgres
# options: >-
# --health-cmd pg_isready
# --health-interval 10s
# --health-timeout 5s
# --health-retries 5
# ports:
# - 5432:5432
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
#
# - uses: actions/cache@v3
# with:
# path: |
# ~/.cache/pip
# .local/share/hatch/
# key: integration-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
#
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
#
# - name: Install packages and dependencies
# run: |
# python -m pip install uv
# uv pip install --system hatch
# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
#
# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
# run: |
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration
# env:
# AIRFLOW__COSMOS__ENABLE_CACHE_DBT_LS: 0
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }}
# AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }}
# AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }}
# DATABRICKS_HOST: mock
# DATABRICKS_WAREHOUSE_ID: mock
# DATABRICKS_TOKEN: mock
# DATABRICKS_CLUSTER_ID: mock
# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
# POSTGRES_HOST: localhost
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# POSTGRES_DB: postgres
# POSTGRES_SCHEMA: public
# POSTGRES_PORT: 5432
#
# - name: Upload coverage to Github
# uses: actions/upload-artifact@v2
# with:
# name: coverage-integration-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
# path: .coverage
#
# Run-Integration-Tests-Expensive:
# needs: Authorize
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.11"]
# airflow-version: ["2.6"]
#
# services:
# postgres:
# image: postgres
# env:
# POSTGRES_PASSWORD: postgres
# options: >-
# --health-cmd pg_isready
# --health-interval 10s
# --health-timeout 5s
# --health-retries 5
# ports:
# - 5432:5432
#
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
# - uses: actions/cache@v3
# with:
# path: |
# ~/.cache/pip
# .local/share/hatch/
# key: integration-expensive-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
#
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
#
# - name: Install packages and dependencies
# run: |
# python -m pip install uv
# uv pip install --system hatch
# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
#
# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
# run: |
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-setup
# DATABRICKS_UNIQUE_ID="${{github.run_id}}" hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-expensive
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }}
# AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }}
# AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }}
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
# AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
# DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
# POSTGRES_HOST: localhost
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# POSTGRES_DB: postgres
# POSTGRES_SCHEMA: public
# POSTGRES_PORT: 5432
#
# - name: Upload coverage to Github
# uses: actions/upload-artifact@v2
# with:
# name: coverage-integration-expensive-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
# path: .coverage
#
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
# AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }}
# DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }}
#
# Run-Integration-Tests-Sqlite:
# needs: Authorize
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.11"]
# airflow-version: ["2.7"]
#
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
# - uses: actions/cache@v3
# with:
# path: |
# ~/.cache/pip
# .local/share/hatch/
# key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
#
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
#
# - name: Install packages and dependencies
# run: |
# python -m pip install uv
# uv pip install --system hatch
# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
#
# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
# run: |
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }}
# AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }}
# AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }}
# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
# DATABRICKS_CLUSTER_ID: mock
# DATABRICKS_HOST: mock
# DATABRICKS_WAREHOUSE_ID: mock
# DATABRICKS_TOKEN: mock
# POSTGRES_HOST: localhost
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# POSTGRES_DB: postgres
# POSTGRES_SCHEMA: public
# POSTGRES_PORT: 5432
#
# - name: Upload coverage to Github
# uses: actions/upload-artifact@v2
# with:
# name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
# path: .coverage
#
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
#
# Run-Integration-Tests-DBT-1-5-4:
# needs: Authorize
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: [ "3.11" ]
# airflow-version: [ "2.7" ]
# services:
# postgres:
# image: postgres
# env:
# POSTGRES_PASSWORD: postgres
# options: >-
# --health-cmd pg_isready
# --health-interval 10s
# --health-timeout 5s
# --health-retries 5
# ports:
# - 5432:5432
#
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
# - uses: actions/cache@v3
# with:
# path: |
# ~/.cache/pip
# .local/share/hatch/
# key: integration-dbt-1-5-4-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
#
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
#
# - name: Install packages and dependencies
# run: |
# python -m pip install uv
# uv pip install --system hatch
# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
#
# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }}, Python ${{ matrix.python-version }} and dbt 1.5.4
# run: |
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-dbt-1-5-4
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }}
# AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }}
# AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }}
# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
# DATABRICKS_CLUSTER_ID: mock
# DATABRICKS_HOST: mock
# DATABRICKS_WAREHOUSE_ID: mock
# DATABRICKS_TOKEN: mock
# POSTGRES_HOST: localhost
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# POSTGRES_DB: postgres
# POSTGRES_SCHEMA: public
# POSTGRES_PORT: 5432
#
# - name: Upload coverage to Github
# uses: actions/upload-artifact@v2
# with:
# name: coverage-integration-dbt-1-5-4-test-${{ matrix.python-version }}-${{ matrix.airflow-version }}
# path: .coverage
#
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
#
# Run-Performance-Tests:
# needs: Authorize
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.11"]
# airflow-version: ["2.7"]
# num-models: [1, 10, 50, 100]
# services:
# postgres:
# image: postgres
# env:
# POSTGRES_PASSWORD: postgres
# options: >-
# --health-cmd pg_isready
# --health-interval 10s
# --health-timeout 5s
# --health-retries 5
# ports:
# - 5432:5432
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
# - uses: actions/cache@v3
# with:
# path: |
# ~/.cache/pip
# .local/share/hatch/
# key: perf-test-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }}
#
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v4
# with:
# python-version: ${{ matrix.python-version }}
#
# - name: Install packages and dependencies
# run: |
# python -m pip install uv
# uv pip install --system hatch
# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze
#
# - name: Run performance tests against against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }}
# id: run-performance-tests
# run: |
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance-setup
# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-performance
#
# # read the performance results and set them as an env var for the next step
# # format: NUM_MODELS={num_models}\nTIME={end - start}\n
# cat /tmp/performance_results.txt > $GITHUB_STEP_SUMMARY
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }}
# POSTGRES_HOST: localhost
# POSTGRES_USER: postgres
# POSTGRES_PASSWORD: postgres
# POSTGRES_DB: postgres
# POSTGRES_SCHEMA: public
# POSTGRES_PORT: 5432
# MODEL_COUNT: ${{ matrix.num-models }}
# env:
# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/
# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:[email protected]:5432/postgres
# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH
#
# Code-Coverage:
# if: github.event.action != 'labeled'
# needs:
# - Run-Unit-Tests
# - Run-Integration-Tests
# - Run-Integration-Tests-Expensive
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# with:
# ref: ${{ github.event.pull_request.head.sha || github.ref }}
# - name: Set up Python 3.11
# uses: actions/setup-python@v3
# with:
# python-version: "3.11"
# - name: Install coverage
# run: |
# pip3 install coverage
# - name: Download all coverage artifacts
# uses: actions/download-artifact@v2
# with:
# path: ./coverage
# - name: Combine coverage
# run: |
# coverage combine ./coverage/coverage*/.coverage
# coverage report
# coverage xml
# - name: Upload coverage to Codecov
# uses: codecov/codecov-action@v3
# with:
# fail_ci_if_error: true
# token: ${{ secrets.CODECOV_TOKEN }}
# files: coverage.xml