integration_tests.yml

name: Integration Testing

on:
  push:
    branches: [ "main" ]
  pull_request_target:
    branches: [ "main" ]

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: true
      max-parallel: 3
      matrix:
        python-version: [ "3.11"] # "3.10", "3.12"]
        dbt-version: ["1.8.0"] # "1.6.0", , "1.8.0b1"]
        data-platform: ["redshift", "snowflake", "bigquery"]

    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dbt-${{ matrix.data-platform }}~=${{ matrix.dbt-version }}
      run: |
        python -m pip install --upgrade pip
        python -m pip install "dbt-${{ matrix.data-platform }}~=${{ matrix.dbt-version }}" "dbt-core~=${{ matrix.dbt-version }}"
    - name: run integration_tests project on ${{ matrix.data-platform }}
      run: |
        cd integration_tests
        export DBT_PROFILES_DIR=.
        dbt deps --target ${{ matrix.data-platform }}
        dbt seed --full-refresh --target ${{ matrix.data-platform }}
        dbt run --target ${{ matrix.data-platform }}
        dbt run-operation prep_external --target ${{ matrix.data-platform }}
        dbt -d run-operation dbt_external_tables.stage_external_sources --vars 'ext_full_refresh: true' --target ${{ matrix.data-platform }}
        dbt run-operation dbt_external_tables.stage_external_sources --target ${{ matrix.data-platform }}
        dbt test --target ${{ matrix.data-platform }}
      env:

        env: 
        # redshift
        REDSHIFT_HOST: ${{ secrets.REDSHIFT_HOST }}
        REDSHIFT_USER: ${{ secrets.REDSHIFT_USER }}
        DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }}
        REDSHIFT_PORT: ${{ secrets.REDSHIFT_PORT }}
        REDSHIFT_DATABASE: ${{ secrets.REDSHIFT_DATABASE }}
        REDSHIFT_SPECTRUM_IAM_ROLE: ${{ secrets.REDSHIFT_SPECTRUM_IAM_ROLE }}
        REDSHIFT_SCHEMA: "dbt_external_tables_integration_tests_redshift"
        #snowflake
        SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
        SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
        DBT_ENV_SECRET_SNOWFLAKE_PASS: ${{ secrets.DBT_ENV_SECRET_SNOWFLAKE_PASS }}
        SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWFLAKE_WAREHOUSE }}
        SNOWFLAKE_ROLE: ${{ secrets.SNOWFLAKE_ROLE }}
        SNOWFLAKE_DATABASE: ${{ secrets.SNOWFLAKE_DATABASE }}
        SNOWFLAKE_SCHEMA: "dbt_external_tables_integration_tests_snowflake"
        # bigquery
        BIGQUERY_PROJECT: ${{ vars.BIGQUERY_PROJECT }}
        BIGQUERY_SCHEMA: "dbt_external_tables_integration_tests_bigquery"
        BIGQUERY_KEYFILE_JSON: ${{ secrets.BIGQUERY_KEYFILE_JSON }}