Update testing structure for dbt Labs testing support - postgres

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+# **what?**
+# Run tests for <this package> against supported adapters
+
+# **why?**
+# To ensure that <this package> works as expected with all supported adapters
+
+# **when?**
+# On every PR, and every push to master and when manually triggered
+
+name: Package Integration Tests
+
+on:
+    push:
+        branches:
+            - master
+    pull_request:
+    workflow_dispatch:
+
+jobs:
+  run-tests:
+      uses: dbt-labs/dbt-package-testing/.github/workflows/run_tox.yml@v1
+      # this just tests with postgres so no variables need to be passed through.
+      # When it's time to add more adapters you will need to pass through inputs for
+      # the other adapters as shown in the below example for redshift
+    #   with:
+    #     # redshift
+    #     REDSHIFT_HOST: ${{ vars.REDSHIFT_HOST }}
+    #     REDSHIFT_USER: ${{ vars.REDSHIFT_USER }}
+    #     REDSHIFT_DATABASE: ${{ vars.REDSHIFT_DATABASE }}
+    #     REDSHIFT_SCHEMA: "integration_tests_redshift_${{ github.run_number }}"
+    #     REDSHIFT_PORT: ${{ vars.REDSHIFT_PORT }}
+    #   secrets:
+    #     DBT_ENV_SECRET_REDSHIFT_PASS: ${{ secrets.DBT_ENV_SECRET_REDSHIFT_PASS }}

dev-requirements.txt

@@ -2,6 +2,7 @@ pytest
 pytest-parametrization>=2022.2.1
 pre-commit
 mypy
+tox
 
 # MyPy stubs
 types-requests

elementary/monitor/dbt_project/dbt_project.yml

@@ -6,7 +6,7 @@ version: "1.0.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: "elementary"
+profile: "integration_tests"
 
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be

integration_tests/profiles.yml

@@ -0,0 +1,14 @@
+integration_tests:
+  target: postgres
+  outputs:
+    postgres:
+      type: "postgres"
+      host: "{{ env_var('POSTGRES_HOST') }}"
+      user: "{{ env_var('POSTGRES_USER') }}"
+      pass: "{{ env_var('DBT_ENV_SECRET_POSTGRES_PASS') }}"
+      port: "{{ env_var('POSTGRES_PORT') | as_number }}"
+      dbname: "{{ env_var('POSTGRES_DATABASE') }}"
+      schema: "{{ env_var('POSTGRES_SCHEMA') }}"
+      threads: 5
+
+# required format for other adapters can be found at https://github.com/dbt-labs/dbt-package-testing/blob/main/integration_tests/profiles.yml

supported_adapters.env

@@ -0,0 +1 @@
+SUPPORTED_ADAPTERS=postgres

tests/tests_with_db/dbt_project/dbt_project.yml

@@ -1,7 +1,7 @@
 name: "elementary_tests"
 version: "1.0.0"
 config-version: 2
-profile: "elementary_tests"
+profile: "integration_tests"
 
 model-paths: ["models"]
 analysis-paths: ["analyses"]

tox.ini

@@ -0,0 +1,44 @@
+[tox]
+skipsdist = True
+envlist = lint_all, testenv, copyfile
+
+[testenv]
+passenv =
+    # postgres env vars
+    POSTGRES_HOST
+    POSTGRES_USER
+    DBT_ENV_SECRET_POSTGRES_PASS
+    POSTGRES_PORT
+    POSTGRES_DATABASE
+    POSTGRES_SCHEMA
+allowlist_externals =
+    edr
+    cp
+    pip
+deps = 
+  -rdev-requirements.txt
+  -e .
+commands =
+    edr
+    # Create the destination folder if it doesn't exist
+    mkdir -p ~/.dbt
+    # Copy the file to the home directory
+    cp integration_tests/profiles.yml ~/.dbt/profiles.yml
+
+# Postgres integration tests for centralized dbt testing
+# run pytest but skips e2e tests with reports
+[testenv:dbt_integration_postgres]
+changedir = tests
+allowlist_externals = 
+    pytest
+    cp
+    edr
+    mkdir
+skip_install = true
+commands =
+    edr
+    # Create the destination folder if it doesn't exist
+    mkdir -p ~/.dbt
+    # Copy the file to the home directory
+    cp ../integration_tests/profiles.yml ~/.dbt/profiles.yml
+    pytest -v --target postgres --ignore e2e