diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..b5cf5ca7f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +# **what?** +# Run tests for against supported adapters + +# **why?** +# To ensure that works as expected with all supported adapters + +# **when?** +# On every PR, and every push to master and when manually triggered + +name: Package Integration Tests + +on: + push: + branches: + - master + pull_request: + workflow_dispatch: + +jobs: + run-tests: + uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1 + # this just tests with postgres so no variables need to be passed through. + # When it's time to add more adapters you will need to pass through inputs for + # the other adapters as shown in the below example for redshift + # with: + # # redshift + # REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }} + # REDSHIFT_USER: ${{ vars.REDSHIFT_USER }} + # REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }} + # REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}" + # REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }} + # secrets: + # DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }} diff --git a/dev-requirements.txt b/dev-requirements.txt index f41268a0d..a24f4d4f3 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,6 +2,7 @@ pytest pytest-parametrization>=2022.2.1 pre-commit mypy +tox # MyPy stubs types-requests diff --git a/elementary/monitor/dbt_project/dbt_project.yml b/elementary/monitor/dbt_project/dbt_project.yml index ce4e3dbe0..ee2eb6a2b 100644 --- a/elementary/monitor/dbt_project/dbt_project.yml +++ b/elementary/monitor/dbt_project/dbt_project.yml @@ -6,7 +6,7 @@ version: "1.0.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: "elementary" +profile: "integration_tests" # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be diff --git a/integration_tests/profiles.yml b/integration_tests/profiles.yml new file mode 100644 index 000000000..043b9f39e --- /dev/null +++ b/integration_tests/profiles.yml @@ -0,0 +1,14 @@ +integration_tests: + target: postgres + outputs: + postgres: + type: "postgres" + host: "{{ env_var('POSTGRES_HOST') }}" + user: "{{ env_var('POSTGRES_USER') }}" + pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}" + port: "{{ env_var('POSTGRES_PORT') | as_number }}" + dbname: "{{ env_var('POSTGRES_DATABASE') }}" + schema: "{{ env_var('POSTGRES_SCHEMA') }}" + threads: 5 + +# required format for other adapters can be found at https://github.com/dbt-labs/dbt-package-testing/blob/main/integration_tests/profiles.yml diff --git a/supported_adapters.env b/supported_adapters.env new file mode 100644 index 000000000..14c965c93 --- /dev/null +++ b/supported_adapters.env @@ -0,0 +1 @@ +SUPPORTED_ADAPTERS=postgres \ No newline at end of file diff --git a/tests/tests_with_db/dbt_project/dbt_project.yml b/tests/tests_with_db/dbt_project/dbt_project.yml index 34cfa5204..00b7fe0c5 100644 --- a/tests/tests_with_db/dbt_project/dbt_project.yml +++ b/tests/tests_with_db/dbt_project/dbt_project.yml @@ -1,7 +1,7 @@ name: "elementary_tests" version: "1.0.0" config-version: 2 -profile: "elementary_tests" +profile: "integration_tests" model-paths: ["models"] analysis-paths: ["analyses"] diff --git a/tox.ini b/tox.ini new file mode 100644 index 000000000..38f340a14 --- /dev/null +++ b/tox.ini @@ -0,0 +1,44 @@ +[tox] +skipsdist = True +envlist = lint_all, testenv, copyfile + +[testenv] +passenv = + # postgres env vars + POSTGRES_HOST + POSTGRES_USER + DBT_ENV_SECRET_POSTGRES_PASS + POSTGRES_PORT + POSTGRES_DATABASE + POSTGRES_SCHEMA +allowlist_externals = + edr + cp + pip +deps = + -rdev-requirements.txt + -e . +commands = + edr + # Create the destination folder if it doesn't exist + mkdir -p ~/.dbt + # Copy the file to the home directory + cp integration_tests/profiles.yml ~/.dbt/profiles.yml + +# Postgres integration tests for centralized dbt testing +# run pytest but skips e2e tests with reports +[testenv:dbt_integration_postgres] +changedir = tests +allowlist_externals = + pytest + cp + edr + mkdir +skip_install = true +commands = + edr + # Create the destination folder if it doesn't exist + mkdir -p ~/.dbt + # Copy the file to the home directory + cp ../integration_tests/profiles.yml ~/.dbt/profiles.yml + pytest -v --target postgres --ignore e2e \ No newline at end of file