-
Notifications
You must be signed in to change notification settings - Fork 0
176 lines (145 loc) · 7.68 KB
/
ci_test_branch_deployments_end_to_end.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
name: Test Branch Deployments End to End
on:
pull_request:
types: [opened, synchronize, reopened, closed]
concurrency:
# Cancel in-progress deploys to same branch
group: ${{ github.ref }}/branch_deployments
cancel-in-progress: true
env:
DAGSTER_CLOUD_URL: "http://wisemuffin.dagster.cloud"
DAGSTER_CLOUD_API_TOKEN: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
ENABLE_FAST_DEPLOYS: 'true'
PYTHON_VERSION: '3.11'
DAGSTER_CLOUD_FILE: 'dagster_cloud.yaml'
DAGSTER_CLOUD_ORGANIZATION: "wisemuffin"
DAGSTER_PROJECT_NAME: pipeline_nsw_doe
NSW_DOE_DATA_STACK_IN_A_BOX_DAGSTER_PROJECT_DIR: ${{ vars.NSW_DOE_DATA_STACK_IN_A_BOX_DAGSTER_PROJECT_DIR }}
NSW_DOE_DATA_STACK_IN_A_BOX_DBT_PROJECT_DIR: ${{ vars.NSW_DOE_DATA_STACK_IN_A_BOX_DBT_PROJECT_DIR }}
NSW_DOE_DATA_STACK_IN_A_BOX__ENV: ${{ vars.NSW_DOE_DATA_STACK_IN_A_BOX__ENV }}
NSW_DOE_DATA_STACK_IN_A_BOX_DB_PATH_AND_DB: ${{ secrets.NSW_DOE_DATA_STACK_IN_A_BOX_DB_PATH_AND_DB }}
NSW_DOE_DATA_STACK_IN_A_BOX_DB_NAME: ${{ vars.NSW_DOE_DATA_STACK_IN_A_BOX_DB_NAME }}
MOTHERDUCK_TOKEN: ${{ secrets.MOTHERDUCK_TOKEN }}
DATAFOLD_APIKEY: ${{ secrets.DATAFOLD_APIKEY }}
DAGSTER_HOME: ${{ vars.DAGSTER_HOME }}
AWS_ROLE_TO_ASSUME: ${{ secrets.AWS_ROLE_TO_ASSUME }}
S3_BUCKET_METADATA: ${{ secrets.S3_BUCKET_METADATA }}
DESTINATION__DUCKDB__CREDENTIALS: ${{ secrets.DESTINATION__DUCKDB__CREDENTIALS }}
SOURCES__GITHUB__ACCESS_TOKEN: ${{ secrets.SOURCES__GITHUB__ACCESS_TOKEN }}
SOURCES__GOOGLE_ANALYTICS__CREDENTIALS__PROJECT_ID: ${{ secrets.SOURCES__GOOGLE_ANALYTICS__CREDENTIALS__PROJECT_ID }}
SOURCES__GOOGLE_ANALYTICS__CREDENTIALS__CLIENT_EMAIL: ${{ secrets.SOURCES__GOOGLE_ANALYTICS__CREDENTIALS__CLIENT_EMAIL }}
SOURCES__GOOGLE_ANALYTICS__CREDENTIALS__PRIVATE_KEY: ${{ secrets.SOURCES__GOOGLE_ANALYTICS__CREDENTIALS__PRIVATE_KEY }}
SOURCES__GOOGLE_ANALYTICS__PROPERTY_ID: ${{ secrets.SOURCES__GOOGLE_ANALYTICS__PROPERTY_ID }}
TPCH__ENV: ${{ vars.TPCH__ENV}}
TPCH_DBT_PROJECT_DIR: ${{ vars.TPCH_DBT_PROJECT_DIR}}
TPCH_DB_PATH_AND_DB: ${{ secrets.TPCH_DB_PATH_AND_DB}}
TPCH_DB_NAME: ${{ vars.TPCH_DB_NAME}}
jobs:
dagster_cloud_default_deploy:
name: Dagster Serverless Deploy
runs-on: ubuntu-20.04
environment: test
outputs:
build_info: ${{ steps.parse-workspace.outputs.build_info }}
steps:
- name: Setup Python
uses: actions/[email protected]
with:
python-version: "3.11.x"
- name: Prerun Checks
id: prerun
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
- name: Set NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA
run: |
# echo "NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA=pr_full_$(echo ${GITHUB_HEAD_REF} | tr '[:upper:]' '[:lower:]' | sed -e 's/[^a-zA-Z0-9]/_/g')" >> $GITHUB_ENV
NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA=pr_full_$(echo ${GITHUB_HEAD_REF} | tr '[:upper:]' '[:lower:]' | sed -e 's/[^a-zA-Z0-9]/_/g')
echo "NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA=$(echo $NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA)" >> $GITHUB_ENV
# echo schema ${{ env.NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA }} # cant ref env variables in command in same run
echo schema $NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA
- name: check schema
run: echo schema ${{ env.NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA }}
- name: Launch Docker Deploy
if: steps.prerun.outputs.result == 'docker-deploy'
id: parse-workspace
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
dagster_cloud_file: $DAGSTER_CLOUD_FILE
- name: Checkout for Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
path: .
- name: Validate configuration
id: ci-validate
if: steps.prerun.outputs.result == 'pex-deploy'
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
command: "ci check --project-dir . --dagster-cloud-yaml-path ${{ env.DAGSTER_CLOUD_FILE }}"
- name: Initialize build session
id: ci-init
if: steps.prerun.outputs.result == 'pex-deploy'
uses: dagster-io/dagster-cloud-action/actions/utils/[email protected]
with:
project_dir: .
dagster_cloud_yaml_path: ${{ env.DAGSTER_CLOUD_FILE }}
- name: Prepare DBT project for deployment
if: steps.prerun.outputs.result == 'pex-deploy'
# --upgrade-strategy eager picks up newer packages that are required for things to work
run: |
# echo "PYTHONPATH="pwd >> $GITHUB_ENV
python -m pip install uv
uv venv
source .venv/bin/activate
# currently requirements.txt is stored at total project level one level above dagster project
uv pip install -r requirements.txt
cd ./transformation/demo_transformation_scaling_tpch
dbt deps
cd ../..
cd ./transformation/transformation_nsw_doe
dbt deps
cd ../..
cd ./${{ env.NSW_DOE_DATA_STACK_IN_A_BOX_DAGSTER_PROJECT_DIR }}
uv pip install .
dagster-dbt project prepare-for-deployment --file ./${{ env.DAGSTER_PROJECT_NAME }}/project.py
dagster-dbt project prepare-for-deployment --file ./pipeline_nsw_doe_requires_secrets/project.py
dagster-dbt project prepare-for-deployment --file ./demo_pipeline_scaling_tpch/project.py
# The cli command below can be used to manage syncing the prod manifest to branches if state_path is set on the DbtProject
# dagster-cloud ci dagster-dbt project manage-state --file ./${{ env.DAGSTER_PROJECT_NAME }}/project.py
shell: bash
- name: Python Executable Deploy
if: steps.prerun.outputs.result == 'pex-deploy'
uses: dagster-io/dagster-cloud-action/actions/[email protected]
with:
dagster_cloud_file: "$GITHUB_WORKSPACE/$DAGSTER_CLOUD_FILE"
build_output_dir: "$GITHUB_WORKSPACE/build"
python_version: "${{ env.PYTHON_VERSION }}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA: ${{ env.NSW_DOE_DATA_STACK_IN_A_BOX_TARGET_SCHEMA }}
# cant seem to send env to dagster config, for now when doing end to end test will need to set this env in the UI
dagster_cloud_docker_deploy:
name: Docker Deploy
runs-on: ubuntu-20.04
if: needs.dagster_cloud_default_deploy.outputs.build_info
needs: dagster_cloud_default_deploy
strategy:
fail-fast: false
matrix:
location: ${{ fromJSON(needs.dagster_cloud_default_deploy.outputs.build_info) }}
steps:
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.head_ref }}
- name: Build and deploy to Dagster Cloud serverless
uses: dagster-io/dagster-cloud-action/actions/[email protected]
with:
dagster_cloud_api_token: ${{ secrets.DAGSTER_CLOUD_API_TOKEN }}
location: ${{ toJson(matrix.location) }}
base_image: "python:${{ env.PYTHON_VERSION }}-slim"
# Uncomment to pass through Github Action secrets as a JSON string of key-value pairs
# env_vars: ${{ toJson(secrets) }}
organization_id: ${{ secrets.ORGANIZATION_ID }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}