From ed485baa4ab127ff6930f9679de178d19c7a376e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 12:21:02 -0400 Subject: [PATCH 01/81] Initial commit. A simple clone of the DB --- .github/workflows/test_with_clone.yaml | 52 ++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .github/workflows/test_with_clone.yaml diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml new file mode 100644 index 00000000..a6f8a1e8 --- /dev/null +++ b/.github/workflows/test_with_clone.yaml @@ -0,0 +1,52 @@ +--- +name: Test Changes with Cloned DB + +on: + push: + branches: "main" + pull_request: + types: [ labeled ] + +permissions: + contents: read + +jobs: + test_with_clone: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'test_with_clone') + env: + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWSQL_PWD }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }} + # TODO: Not the right role? + SNOWFLAKE_ROLE: DATA_ENGINEER + # TODO: This needs to be the dev db, since that is the bleeding edge + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} + SNOWFLAKE_CLONED_DATABASE_NAME: "synapse_data_warehouse_${{ github.sha }}" + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} + SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} + STACK: ${{ vars.STACK }} + + steps: + + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: install-py-dependencies + shell: bash + run: | + pip install schemachange==3.6.1 + pip install numpy==1.26.4 + pip install pandas==1.5.3 + + # We clone the DB using the syntax seen here: + # https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas + - name: zero-copy clone the database + shell: bash + # TODO: How do we drop the tasks? Add "IGNORE TASKS" at the end? This is maybe only relevant if we use SYSADMIN role for cloning. + run: | + ~/bin/snowsql -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE \ + -q "CREATE DATABASE $SNOWFLAKE_CLONED_DATABASE_NAME CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE" \ No newline at end of file From b3889154cc4f4112a345dde7d2ccafdcc5ee8f34 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 12:33:04 -0400 Subject: [PATCH 02/81] Install snowsql --- .github/workflows/test_with_clone.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index a6f8a1e8..e0b82500 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,6 +42,11 @@ jobs: pip install numpy==1.26.4 pip install pandas==1.5.3 + - name: Install SnowSQL + run: | + curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash + SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + # We clone the DB using the syntax seen here: # https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas - name: zero-copy clone the database From e3c575a3e9ebb675e3ae8ff46761ca9f81108a12 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 12:35:25 -0400 Subject: [PATCH 03/81] add pw --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index e0b82500..2fce05cd 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -53,5 +53,5 @@ jobs: shell: bash # TODO: How do we drop the tasks? Add "IGNORE TASKS" at the end? This is maybe only relevant if we use SYSADMIN role for cloning. run: | - ~/bin/snowsql -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE \ + ~/bin/snowsql -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -p $SNOWFLAKE_PASSWORD \ -q "CREATE DATABASE $SNOWFLAKE_CLONED_DATABASE_NAME CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE" \ No newline at end of file From ae8d75dd15e1fc275b091db753299a7a74f65b0a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 12:44:41 -0400 Subject: [PATCH 04/81] add debug step --- .github/workflows/test_with_clone.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 2fce05cd..488d2288 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -47,6 +47,11 @@ jobs: curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + - name: debug + run: | + echo "SNOWFLAKE_ACCOUNT: $SNOWFLAKE_ACCOUNT" + echo "SNOWFLAKE_USER: $SNOWFLAKE_USER" + echo "SNOWFLAKE_ROLE: $SNOWFLAKE_ROLE" # We clone the DB using the syntax seen here: # https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas - name: zero-copy clone the database From 4371e0bf601724557cf8de17acede32202daabe1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 12:51:08 -0400 Subject: [PATCH 05/81] . --- .github/workflows/test_with_clone.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 488d2288..1d0ccb36 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -48,10 +48,12 @@ jobs: SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - name: debug + shell: bash run: | echo "SNOWFLAKE_ACCOUNT: $SNOWFLAKE_ACCOUNT" echo "SNOWFLAKE_USER: $SNOWFLAKE_USER" echo "SNOWFLAKE_ROLE: $SNOWFLAKE_ROLE" + # We clone the DB using the syntax seen here: # https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas - name: zero-copy clone the database From c9989e50b30c00b0510b952f275848e965fe7e86 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 13:01:30 -0400 Subject: [PATCH 06/81] debugging --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 1d0ccb36..c0bac4c5 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -51,7 +51,7 @@ jobs: shell: bash run: | echo "SNOWFLAKE_ACCOUNT: $SNOWFLAKE_ACCOUNT" - echo "SNOWFLAKE_USER: $SNOWFLAKE_USER" + echo "SNOWFLAKE_USER: secrets.SNOWSQL_USER" echo "SNOWFLAKE_ROLE: $SNOWFLAKE_ROLE" # We clone the DB using the syntax seen here: From 1b136f039f6098d6258c9d6d16056fd31ec38997 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 13:02:53 -0400 Subject: [PATCH 07/81] syntax --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index c0bac4c5..3cf87ed3 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -51,7 +51,7 @@ jobs: shell: bash run: | echo "SNOWFLAKE_ACCOUNT: $SNOWFLAKE_ACCOUNT" - echo "SNOWFLAKE_USER: secrets.SNOWSQL_USER" + echo "SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }}" echo "SNOWFLAKE_ROLE: $SNOWFLAKE_ROLE" # We clone the DB using the syntax seen here: From 680ecaa1e2f73caca21693c4397ba62810d7c8a6 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 13:06:26 -0400 Subject: [PATCH 08/81] directly pass vars --- .github/workflows/test_with_clone.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 3cf87ed3..a8999838 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -58,7 +58,18 @@ jobs: # https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas - name: zero-copy clone the database shell: bash + # TODO: How do we drop the tasks? Add "IGNORE TASKS" at the end? This is maybe only relevant if we use SYSADMIN role for cloning. run: | - ~/bin/snowsql -a $SNOWFLAKE_ACCOUNT -u $SNOWFLAKE_USER -r $SNOWFLAKE_ROLE -w $SNOWFLAKE_WAREHOUSE -p $SNOWFLAKE_PASSWORD \ + env SNOWFLAKE_ACCOUNT=$SNOWFLAKE_ACCOUNT \ + SNOWFLAKE_USER=$SNOWFLAKE_USER \ + SNOWFLAKE_ROLE=$SNOWFLAKE_ROLE \ + SNOWFLAKE_WAREHOUSE=$SNOWFLAKE_WAREHOUSE \ + SNOWFLAKE_PASSWORD=$SNOWFLAKE_PASSWORD \ + ~/bin/snowsql \ + -a $SNOWFLAKE_ACCOUNT \ + -u $SNOWFLAKE_USER \ + -r $SNOWFLAKE_ROLE \ + -w $SNOWFLAKE_WAREHOUSE \ + -p $SNOWFLAKE_PASSWORD \ -q "CREATE DATABASE $SNOWFLAKE_CLONED_DATABASE_NAME CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE" \ No newline at end of file From 7d946edb01cb981f93e2fa783a9c31a50cac6acb Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 13:37:51 -0400 Subject: [PATCH 09/81] new .sql file to run the command isntead --- .github/workflows/test_with_clone.yaml | 17 +---------------- admin/clone_db.sql | 5 +++++ 2 files changed, 6 insertions(+), 16 deletions(-) create mode 100644 admin/clone_db.sql diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index a8999838..2358ca73 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -54,22 +54,7 @@ jobs: echo "SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }}" echo "SNOWFLAKE_ROLE: $SNOWFLAKE_ROLE" - # We clone the DB using the syntax seen here: - # https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas - name: zero-copy clone the database shell: bash - - # TODO: How do we drop the tasks? Add "IGNORE TASKS" at the end? This is maybe only relevant if we use SYSADMIN role for cloning. run: | - env SNOWFLAKE_ACCOUNT=$SNOWFLAKE_ACCOUNT \ - SNOWFLAKE_USER=$SNOWFLAKE_USER \ - SNOWFLAKE_ROLE=$SNOWFLAKE_ROLE \ - SNOWFLAKE_WAREHOUSE=$SNOWFLAKE_WAREHOUSE \ - SNOWFLAKE_PASSWORD=$SNOWFLAKE_PASSWORD \ - ~/bin/snowsql \ - -a $SNOWFLAKE_ACCOUNT \ - -u $SNOWFLAKE_USER \ - -r $SNOWFLAKE_ROLE \ - -w $SNOWFLAKE_WAREHOUSE \ - -p $SNOWFLAKE_PASSWORD \ - -q "CREATE DATABASE $SNOWFLAKE_CLONED_DATABASE_NAME CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE" \ No newline at end of file + ~/bin/snowsql -f admin/clone_db.sql \ No newline at end of file diff --git a/admin/clone_db.sql b/admin/clone_db.sql new file mode 100644 index 00000000..ca5b3868 --- /dev/null +++ b/admin/clone_db.sql @@ -0,0 +1,5 @@ +USE ROLE DATA_ENGINEER; + +-- We clone the DB using the syntax seen here: +-- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas +CREATE DATABASE TEST_CLONE_DB CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From a342f8583fcb1f8e8277fc3cdb9c8d48f02c3e3f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 13:49:39 -0400 Subject: [PATCH 10/81] global vars --- .github/workflows/test_with_clone.yaml | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 2358ca73..28997b8c 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -15,18 +15,10 @@ jobs: runs-on: ubuntu-latest if: contains(github.event.pull_request.labels.*.name, 'test_with_clone') env: - SNOWFLAKE_PASSWORD: ${{ secrets.SNOWSQL_PWD }} - SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} - SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }} - # TODO: Not the right role? - SNOWFLAKE_ROLE: DATA_ENGINEER - # TODO: This needs to be the dev db, since that is the bleeding edge - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} - SNOWFLAKE_CLONED_DATABASE_NAME: "synapse_data_warehouse_${{ github.sha }}" - SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} - SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} - STACK: ${{ vars.STACK }} + SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} + SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} + SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} steps: @@ -50,9 +42,9 @@ jobs: - name: debug shell: bash run: | - echo "SNOWFLAKE_ACCOUNT: $SNOWFLAKE_ACCOUNT" - echo "SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }}" - echo "SNOWFLAKE_ROLE: $SNOWFLAKE_ROLE" + echo "SNOWFLAKE_ACCOUNT: $SNOWSQL_ACCOUNT" + echo "SNOWFLAKE_USER: $SNOWSQL_USER" + echo "SNOWFLAKE_WH: $SNOWSQL_WAREHOUSE" - name: zero-copy clone the database shell: bash From 244216b9f3cdb5344671e242e8ec0b60d0f8c859 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 13:55:57 -0400 Subject: [PATCH 11/81] dynamically change the name of the clone DB --- .github/workflows/test_with_clone.yaml | 9 +-------- admin/clone_db.sql | 2 +- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 28997b8c..7402a9da 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -39,14 +39,7 @@ jobs: curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - - name: debug - shell: bash - run: | - echo "SNOWFLAKE_ACCOUNT: $SNOWSQL_ACCOUNT" - echo "SNOWFLAKE_USER: $SNOWSQL_USER" - echo "SNOWFLAKE_WH: $SNOWSQL_WAREHOUSE" - - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql -f admin/clone_db.sql \ No newline at end of file + ~/bin/snowsql --variable CLONED_DB_NAME="SYNAPSE_DATA_WAREHOUSE_${GITHUB_SHA:0:8}" -f admin/clone_db.sql \ No newline at end of file diff --git a/admin/clone_db.sql b/admin/clone_db.sql index ca5b3868..c0f02cc4 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,4 +2,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE TEST_CLONE_DB CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER($CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From 4f35760da47bf256d0015c7bb9d9fa14f31eba29 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:04:18 -0400 Subject: [PATCH 12/81] syntax --- .github/workflows/test_with_clone.yaml | 2 +- admin/clone_db.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 7402a9da..196a637f 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql --variable CLONED_DB_NAME="SYNAPSE_DATA_WAREHOUSE_${GITHUB_SHA:0:8}" -f admin/clone_db.sql \ No newline at end of file + ~/bin/snowsql -f admin/clone_db.sql --variable CLONED_DB_NAME="SYNAPSE_DATA_WAREHOUSE_${GITHUB_SHA:0:8}" \ No newline at end of file diff --git a/admin/clone_db.sql b/admin/clone_db.sql index c0f02cc4..be77d793 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,4 +2,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER($CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE '&CLONED_DB_NAME' CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From ab2a04892b9323896088fa80b7f5846d1eb8e85f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:06:57 -0400 Subject: [PATCH 13/81] syntax --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index be77d793..60ffc1a7 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,4 +2,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE '&CLONED_DB_NAME' CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE &CLONED_DB_NAME CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From be2e82f2dcdcef2d125110ee16fafb840fc1c2dc Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:11:26 -0400 Subject: [PATCH 14/81] using IDENTIFIER for db object --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 60ffc1a7..f917b7ad 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,4 +2,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE &CLONED_DB_NAME CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER('&CLONED_DB_NAME') CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From fb4a20896ba03c898b94d4ac82e168506429045e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:13:38 -0400 Subject: [PATCH 15/81] . --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index f917b7ad..c0f02cc4 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,4 +2,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER('&CLONED_DB_NAME') CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER($CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From 913ef4c525170ab4189693ce093cd633953a92c1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:14:57 -0400 Subject: [PATCH 16/81] . --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index c0f02cc4..7fde9982 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,4 +2,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER($CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER(&CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From ded7408e0210be4e7cdbef5b20e37027bf5b03c8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:24:36 -0400 Subject: [PATCH 17/81] set var sub true --- admin/clone_db.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 7fde9982..25bffd78 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -1,5 +1,7 @@ +!set variable_substitution=true; --noqa: PRS + USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER(&CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER('&CLONED_DB_NAME') CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From b25b10d7f2c6dc1ca7331745828105e259f8b235 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:26:57 -0400 Subject: [PATCH 18/81] add identifier --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 25bffd78..a0a8e9be 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -4,4 +4,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER('&CLONED_DB_NAME') CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER(&CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From 2bdc6ef639b7ba05d39dc607dd757427b9572152 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 14:35:01 -0400 Subject: [PATCH 19/81] . --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index a0a8e9be..36ec3b15 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -4,4 +4,4 @@ USE ROLE DATA_ENGINEER; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER(&CLONED_DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER('${CLONED_DB_NAME}') CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From 2090fadb5245f0a1be9733755ba913c7ecc55829 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:34:22 -0400 Subject: [PATCH 20/81] . --- .github/workflows/test_with_clone.yaml | 2 +- admin/clone_db.sql | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 196a637f..4da4845d 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql -f admin/clone_db.sql --variable CLONED_DB_NAME="SYNAPSE_DATA_WAREHOUSE_${GITHUB_SHA:0:8}" \ No newline at end of file + ~/bin/snowsql --variable CLONED_DB_NAME="test123" -f admin/clone_db.sql \ No newline at end of file diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 36ec3b15..3cfa254b 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,6 +2,8 @@ USE ROLE DATA_ENGINEER; +SET DB_NAME = CLONED_DB_NAME + -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER('${CLONED_DB_NAME}') CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER(DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From b0ebf0c4f11f3b3f00b391e1077c266666cefeb3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:35:27 -0400 Subject: [PATCH 21/81] typo --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 3cfa254b..a629a2dc 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,7 +2,7 @@ USE ROLE DATA_ENGINEER; -SET DB_NAME = CLONED_DB_NAME +SET DB_NAME = CLONED_DB_NAME; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas From d91d5ab8c42d191696a45239b0cbf46bb12c3356 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:37:05 -0400 Subject: [PATCH 22/81] . --- .github/workflows/test_with_clone.yaml | 2 +- admin/clone_db.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 4da4845d..afab06d7 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql --variable CLONED_DB_NAME="test123" -f admin/clone_db.sql \ No newline at end of file + ~/bin/snowsql --variable cloned_db_name="test123" -f admin/clone_db.sql \ No newline at end of file diff --git a/admin/clone_db.sql b/admin/clone_db.sql index a629a2dc..5cd6960b 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,7 +2,7 @@ USE ROLE DATA_ENGINEER; -SET DB_NAME = CLONED_DB_NAME; +SET DB_NAME = :cloned_db_name; -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas From 6a7b631c0fd1d39e1186029073a531c1c32c0917 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:41:07 -0400 Subject: [PATCH 23/81] . --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index afab06d7..645021f4 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql --variable cloned_db_name="test123" -f admin/clone_db.sql \ No newline at end of file + ~/bin/snowsql -f admin/clone_db.sql --variable cloned_db_name="test123" \ No newline at end of file From d8c892719a0aa6e61bffa85e07a11bd71832dd5a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:47:47 -0400 Subject: [PATCH 24/81] ? --- .github/workflows/test_with_clone.yaml | 2 +- admin/clone_db.sql | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 645021f4..87dffd04 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql -f admin/clone_db.sql --variable cloned_db_name="test123" \ No newline at end of file + ~/bin/snowsql -f admin/clone_db.sql --variable cloned_db_name=test123 \ No newline at end of file diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 5cd6960b..4c893fdb 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -2,8 +2,6 @@ USE ROLE DATA_ENGINEER; -SET DB_NAME = :cloned_db_name; - -- We clone the DB using the syntax seen here: -- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER(DB_NAME) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file +CREATE DATABASE IDENTIFIER($cloned_db_name) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From 38da909986a14517b0b43c0073fc43d9e74db879 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:50:30 -0400 Subject: [PATCH 25/81] . --- admin/clone_db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin/clone_db.sql b/admin/clone_db.sql index 4c893fdb..abcb34d6 100644 --- a/admin/clone_db.sql +++ b/admin/clone_db.sql @@ -1,4 +1,4 @@ -!set variable_substitution=true; --noqa: PRS +!set variable_substitution=true; USE ROLE DATA_ENGINEER; From c75abbaa024e2539f59d8d571ebea4a4a31f94b2 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 16:59:20 -0400 Subject: [PATCH 26/81] go back to running command in ci step itself --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 87dffd04..2b646ce3 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql -f admin/clone_db.sql --variable cloned_db_name=test123 \ No newline at end of file + ~/bin/snowsql -q "CREATE DATABASE test_12 CLONE SYNAPSE_DATA_WAREHOUSE_DEV;" \ No newline at end of file From 1a5160f09c400edf8ce460eebcc39b92cb2216b3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 17:02:48 -0400 Subject: [PATCH 27/81] role is data_engineer --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 2b646ce3..d51718e5 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,4 +42,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql -q "CREATE DATABASE test_12 CLONE SYNAPSE_DATA_WAREHOUSE_DEV;" \ No newline at end of file + ~/bin/snowsql -r DATA_ENGINEER -q "CREATE DATABASE test_12 CLONE SYNAPSE_DATA_WAREHOUSE_DEV;" \ No newline at end of file From 1edbb832a819f5876b39c0f2005254b339c563bc Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 23 Oct 2024 17:11:12 -0400 Subject: [PATCH 28/81] branch sha --- .github/workflows/test_with_clone.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index d51718e5..33f4bd4f 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -19,6 +19,8 @@ jobs: SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWSQL_ROLE: DATA_ENGINEER + CLONED_DB_NAME: "synapse_data_warehouse_${{ github.sha }}" steps: @@ -42,4 +44,4 @@ jobs: - name: zero-copy clone the database shell: bash run: | - ~/bin/snowsql -r DATA_ENGINEER -q "CREATE DATABASE test_12 CLONE SYNAPSE_DATA_WAREHOUSE_DEV;" \ No newline at end of file + ~/bin/snowsql -r $SNOWSQL_ROLE -q "CREATE DATABASE $CLONED_DB_NAME CLONE SYNAPSE_DATA_WAREHOUSE_DEV;" \ No newline at end of file From 1bc7f0403470f471ab37c845f87c07ad5bcf2921 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 24 Oct 2024 10:45:37 -0400 Subject: [PATCH 29/81] Run schemachagne on the clone --- .github/workflows/test_with_clone.yaml | 29 +++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 33f4bd4f..20139758 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -14,13 +14,17 @@ jobs: test_with_clone: runs-on: ubuntu-latest if: contains(github.event.pull_request.labels.*.name, 'test_with_clone') + environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWSQL_ROLE: DATA_ENGINEER - CLONED_DB_NAME: "synapse_data_warehouse_${{ github.sha }}" + SNOWSQL_CLONE_ROLE: DATA_ENGINEER + #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_CLONE: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.sha }}" + steps: @@ -29,7 +33,7 @@ jobs: with: python-version: '3.10' - - name: install-py-dependencies + - name: Install python libraries shell: bash run: | pip install schemachange==3.6.1 @@ -41,7 +45,22 @@ jobs: curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - - name: zero-copy clone the database + - name: Verify SnowSQL installation + run: ~/bin/snowsql -v + + - name: Zero-copy clone the database + shell: bash + run: | + ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_CLONE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE;" + + # TODO: Do we need SYSADMIN to run schemachange? + - name: Run schemachange on the clone shell: bash run: | - ~/bin/snowsql -r $SNOWSQL_ROLE -q "CREATE DATABASE $CLONED_DB_NAME CLONE SYNAPSE_DATA_WAREHOUSE_DEV;" \ No newline at end of file + schemachange \ + -f synapse_data_warehouse \ + -a $SNOWFLAKE_ACCOUNT \ + -u $SNOWFLAKE_USER \ + -r $SNOWSQL_CLONE_ROLE \ + -w compute_xsmall \ + --config-folder synapse_data_warehouse From a94ea31ea2d93a61e86fbf88f68038aa1b4f1b2d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 24 Oct 2024 10:48:53 -0400 Subject: [PATCH 30/81] . --- .github/workflows/test_with_clone.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 20139758..d461e978 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -22,8 +22,8 @@ jobs: SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWSQL_CLONE_ROLE: DATA_ENGINEER #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_CLONE: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.sha }}" + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.sha }}" + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} steps: @@ -51,7 +51,7 @@ jobs: - name: Zero-copy clone the database shell: bash run: | - ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_CLONE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE;" + ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" # TODO: Do we need SYSADMIN to run schemachange? - name: Run schemachange on the clone From cee2c5ea51903b74bc9b2abdfd5d8b31475b402d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 24 Oct 2024 10:53:45 -0400 Subject: [PATCH 31/81] . --- .github/workflows/test_with_clone.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index d461e978..1d274161 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -2,8 +2,6 @@ name: Test Changes with Cloned DB on: - push: - branches: "main" pull_request: types: [ labeled ] From df3a91fb8f081f326e2a62aabf2bea25ab461b0f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 24 Oct 2024 15:08:35 -0400 Subject: [PATCH 32/81] remove clone_db.sql --- admin/clone_db.sql | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 admin/clone_db.sql diff --git a/admin/clone_db.sql b/admin/clone_db.sql deleted file mode 100644 index abcb34d6..00000000 --- a/admin/clone_db.sql +++ /dev/null @@ -1,7 +0,0 @@ -!set variable_substitution=true; - -USE ROLE DATA_ENGINEER; - --- We clone the DB using the syntax seen here: --- https://docs.snowflake.com/en/sql-reference/sql/create-clone#databases-schemas -CREATE DATABASE IDENTIFIER($cloned_db_name) CLONE SYNAPSE_DATA_WAREHOUSE_DEV; \ No newline at end of file From 46c8390edbfa70d95f269dd1e5ac611f05e07675 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 25 Oct 2024 16:42:11 -0400 Subject: [PATCH 33/81] label name changed --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 1d274161..0cafe312 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -11,7 +11,7 @@ permissions: jobs: test_with_clone: runs-on: ubuntu-latest - if: contains(github.event.pull_request.labels.*.name, 'test_with_clone') + if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} From ca73fb60b613f94b321f8f313f6b0c41422bc80e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 12:20:32 -0400 Subject: [PATCH 34/81] change job name. change clone name convention. fix typos. new delete_clone job. --- .github/workflows/test_with_clone.yaml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 0cafe312..7ffc51ae 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -9,7 +9,7 @@ permissions: contents: read jobs: - test_with_clone: + create_clone_and_run_schemachange: runs-on: ubuntu-latest if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') environment: dev @@ -20,7 +20,7 @@ jobs: SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWSQL_CLONE_ROLE: DATA_ENGINEER #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.sha }}" + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} @@ -49,7 +49,7 @@ jobs: - name: Zero-copy clone the database shell: bash run: | - ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" + ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" # TODO: Do we need SYSADMIN to run schemachange? - name: Run schemachange on the clone @@ -57,8 +57,13 @@ jobs: run: | schemachange \ -f synapse_data_warehouse \ - -a $SNOWFLAKE_ACCOUNT \ - -u $SNOWFLAKE_USER \ + -a $SNOWSQL_ACCOUNT \ + -u $SNOWSQL_USER \ -r $SNOWSQL_CLONE_ROLE \ - -w compute_xsmall \ + -w $SNOWSQL_WAREHOUSE \ --config-folder synapse_data_warehouse + + # delete_clone: + # runs-on: ubuntu-latest + # if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') + # environment: dev \ No newline at end of file From 6fe39caba5c8cf628e3272e9d83a9001a5bfc9d7 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 12:39:34 -0400 Subject: [PATCH 35/81] address syntax issues in clone name --- .github/workflows/test_with_clone.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 7ffc51ae..12c3d6dc 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -20,7 +20,7 @@ jobs: SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWSQL_CLONE_ROLE: DATA_ENGINEER #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" + CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} @@ -46,6 +46,11 @@ jobs: - name: Verify SnowSQL installation run: ~/bin/snowsql -v + - name: Sanitize Clone Name + run: | + CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" + echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV + - name: Zero-copy clone the database shell: bash run: | From 88c7b00e87898cde98e23a46b477e0d0e35e6020 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 12:42:29 -0400 Subject: [PATCH 36/81] add a status message --- .github/workflows/test_with_clone.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 12c3d6dc..5f5f7a94 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -50,6 +50,7 @@ jobs: run: | CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV + echo "Clone name has been updated! The clone name will be: ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}" - name: Zero-copy clone the database shell: bash From 939c9f7395d29fb80e05437f54f913ed7c34cf5e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 14:06:18 -0400 Subject: [PATCH 37/81] missing env var --- .github/workflows/test_with_clone.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 5f5f7a94..7410a2cd 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -20,8 +20,9 @@ jobs: SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWSQL_CLONE_ROLE: DATA_ENGINEER #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} + SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} + CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" steps: From 4116019fcc7ce4f85dd1518341ae139da6072b40 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 14:14:22 -0400 Subject: [PATCH 38/81] vars --- .github/workflows/test_with_clone.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 7410a2cd..6f5dd7e5 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -22,8 +22,9 @@ jobs: #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} + SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" - + STACK: ${{ vars.STACK }} steps: From bac91b8e7041afa6a4f65c80a82798c1e8a66aeb Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 14:19:34 -0400 Subject: [PATCH 39/81] sysadmin to run schemachange --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 6f5dd7e5..7a4e0ad0 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -67,7 +67,7 @@ jobs: -f synapse_data_warehouse \ -a $SNOWSQL_ACCOUNT \ -u $SNOWSQL_USER \ - -r $SNOWSQL_CLONE_ROLE \ + -r SYSADMIN \ -w $SNOWSQL_WAREHOUSE \ --config-folder synapse_data_warehouse From 1ffafa900c8fb6f4b3bc68e2c49af2f0825b8859 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 14:22:17 -0400 Subject: [PATCH 40/81] syntax fix --- .github/workflows/test_with_clone.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 7a4e0ad0..2c9a7148 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -53,6 +53,7 @@ jobs: CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV echo "Clone name has been updated! The clone name will be: ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}" + echo $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE - name: Zero-copy clone the database shell: bash From c44ae9e6cab59a025852904bbeb62e566c684958 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 15:49:28 -0400 Subject: [PATCH 41/81] refactor and introduce drop_clone job --- .github/workflows/test_with_clone.yaml | 108 +++++++++++++++---------- 1 file changed, 64 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 2c9a7148..9155a739 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -8,59 +8,69 @@ on: permissions: contents: read +# We define global environment variables needed to run the SnowSQL and schemachange commands +env: + SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} + SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} + SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWSQL_CLONE_ROLE: DATA_ENGINEER + SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} + SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} + SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} + CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" + STACK: ${{ vars.STACK }} + jobs: + setup_environment: + runs-on: ubuntu-latest + environment: dev + outputs: + clone_name: ${{ steps.sanitize_clone.outputs.clone_name }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install python libraries + shell: bash + run: | + pip install schemachange==3.6.1 + pip install numpy==1.26.4 + pip install pandas==1.5.3 + + - name: Install SnowSQL + run: | + curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash + SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + + - name: Verify SnowSQL installation + run: ~/bin/snowsql -v + + - name: Sanitize Clone Name + id: sanitize_clone + run: | + CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" + echo "::set-output name=clone_name::${CLONE_NAME_SANITIZED}" + echo "Clone name has been updated! The clone name will be: ${CLONE_NAME_SANITIZED}" + create_clone_and_run_schemachange: runs-on: ubuntu-latest + needs: setup_environment if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') environment: dev env: - SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} - SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} - SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} - SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWSQL_CLONE_ROLE: DATA_ENGINEER - #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} - SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} - SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} - CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" - STACK: ${{ vars.STACK }} + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ needs.setup_environment.outputs.clone_name }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install python libraries - shell: bash - run: | - pip install schemachange==3.6.1 - pip install numpy==1.26.4 - pip install pandas==1.5.3 - - - name: Install SnowSQL - run: | - curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash - SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - - - name: Verify SnowSQL installation - run: ~/bin/snowsql -v - - - name: Sanitize Clone Name - run: | - CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" - echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV - echo "Clone name has been updated! The clone name will be: ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}" - echo $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE - - name: Zero-copy clone the database shell: bash run: | ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" - # TODO: Do we need SYSADMIN to run schemachange? - name: Run schemachange on the clone shell: bash run: | @@ -68,11 +78,21 @@ jobs: -f synapse_data_warehouse \ -a $SNOWSQL_ACCOUNT \ -u $SNOWSQL_USER \ - -r SYSADMIN \ + -r $SNOWSQL_SCHEMACHANGE_ROLE \ -w $SNOWSQL_WAREHOUSE \ --config-folder synapse_data_warehouse - # delete_clone: - # runs-on: ubuntu-latest - # if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') - # environment: dev \ No newline at end of file + drop_clone: + runs-on: ubuntu-latest + needs: setup_environment + if: contains(github.event.pull_request.labels.*.name, 'drop_clone') + environment: dev + env: + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ needs.setup_environment.outputs.clone_name }} + + steps: + + - name: Drop the clone + shell: bash + run: | + ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "DROP DATABASE IF EXISTS $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE;" \ No newline at end of file From a3eade512e32f992e4c352ce9eb9f756b931e5e6 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 16:21:52 -0400 Subject: [PATCH 42/81] no refactoring --- .github/workflows/test_with_clone.yaml | 115 +++++++++++++++++-------- 1 file changed, 78 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 9155a739..f08aeeb4 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -23,49 +23,54 @@ env: STACK: ${{ vars.STACK }} jobs: - setup_environment: - runs-on: ubuntu-latest - environment: dev - outputs: - clone_name: ${{ steps.sanitize_clone.outputs.clone_name }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install python libraries - shell: bash - run: | - pip install schemachange==3.6.1 - pip install numpy==1.26.4 - pip install pandas==1.5.3 - - - name: Install SnowSQL - run: | - curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash - SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - - - name: Verify SnowSQL installation - run: ~/bin/snowsql -v - - - name: Sanitize Clone Name - id: sanitize_clone - run: | - CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" - echo "::set-output name=clone_name::${CLONE_NAME_SANITIZED}" - echo "Clone name has been updated! The clone name will be: ${CLONE_NAME_SANITIZED}" - + create_clone_and_run_schemachange: runs-on: ubuntu-latest - needs: setup_environment if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') environment: dev env: - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ needs.setup_environment.outputs.clone_name }} + SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} + SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} + SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWSQL_CLONE_ROLE: DATA_ENGINEER + # We need to run schemachange on SYSADMIN for it to work + SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} + SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} + SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} + CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" + STACK: ${{ vars.STACK }} steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install python libraries + shell: bash + run: | + pip install schemachange==3.6.1 + pip install numpy==1.26.4 + pip install pandas==1.5.3 + + - name: Install SnowSQL + run: | + curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash + SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + + - name: Verify SnowSQL installation + run: ~/bin/snowsql -v + + - name: Sanitize Clone Name + run: | + CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" + echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV + echo "Clone name has been updated! The clone name will be: ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}" + echo $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE + - name: Zero-copy clone the database shell: bash run: | @@ -84,14 +89,50 @@ jobs: drop_clone: runs-on: ubuntu-latest - needs: setup_environment if: contains(github.event.pull_request.labels.*.name, 'drop_clone') environment: dev env: - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE: ${{ needs.setup_environment.outputs.clone_name }} + SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} + SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} + SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWSQL_CLONE_ROLE: DATA_ENGINEER + #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN + SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} + SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} + SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} + CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" + STACK: ${{ vars.STACK }} steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install python libraries + shell: bash + run: | + pip install schemachange==3.6.1 + pip install numpy==1.26.4 + pip install pandas==1.5.3 + + - name: Install SnowSQL + run: | + curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash + SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + + - name: Verify SnowSQL installation + run: ~/bin/snowsql -v + + - name: Sanitize Clone Name + run: | + CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" + echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV + echo "Clone name has been updated! The clone name will be: ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}" + echo $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE + - name: Drop the clone shell: bash run: | From d76999f9fe2989e3d1b93d34bcc4fbed34d3e478 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 16:54:31 -0400 Subject: [PATCH 43/81] drop_clone will only run after PR merging --- .github/workflows/test_with_clone.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index f08aeeb4..16a47ef1 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -3,7 +3,7 @@ name: Test Changes with Cloned DB on: pull_request: - types: [ labeled ] + types: [ labeled, closed ] permissions: contents: read @@ -67,9 +67,8 @@ jobs: - name: Sanitize Clone Name run: | CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" + echo "Clone name has been updated! The clone name will be: ${CLONE_NAME_SANITIZED}" echo "SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE=${CLONE_NAME_SANITIZED}" >> $GITHUB_ENV - echo "Clone name has been updated! The clone name will be: ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}" - echo $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE - name: Zero-copy clone the database shell: bash @@ -89,7 +88,7 @@ jobs: drop_clone: runs-on: ubuntu-latest - if: contains(github.event.pull_request.labels.*.name, 'drop_clone') + if: github.event.pull_request.merged == true environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} From 89f06b4df3dc4c8a7a29ef27f1c5e688d99529eb Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 19:12:44 -0400 Subject: [PATCH 44/81] new contributing.md --- CONTRIBUTING.md | 86 +++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..51ab3d0e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,86 @@ +# Contributing Guidelines + +Welcome to the Snowflake repository! Please follow these guidelines to ensure a smooth development workflow and maintain the quality of our shared codebase. + +## Getting Started + +To start contributing, follow these steps to set up your local repository: + +### 1. Clone the Repository: + +```bash +git clone https://github.com/Sage-Bionetworks/snowflake +``` + +### 2. Fetch the Latest `dev` Branch + +After cloning, navigate to the repository directory: + +```bash +cd snowflake +``` + +Then, fetch the latest updates from the `dev` branch to ensure you’re working with the latest codebase: + +```bash +git fetch origin dev +``` + +### 3. Create a New Branch Off `dev` + +Create and checkout your feature branch from the latest `dev` branch. Name it based on your feature or fix, and JIRA ticket number (if applicable). For example: + +```bash +git checkout -b snow-123-new-feature origin/dev +``` + +Your branch will now be tracking `origin/dev` which you can merge with or rebase onto should a merge conflict occur. For more guidance +on how to resolve merge conflicts, [see here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/about-merge-conflicts#resolving-merge-conflicts). + +### 4. Push to The Remote Branch + +Once you've made your changes and committed them locally, push your branch to the remote repository: + +``` +git push origin snow-123-new-feature +``` + +### 5. Create a Draft Pull Request + +In order to initiate automated testing you will need to work on a draft pull request (PR) on GitHub. After pushing your commits to +the remote branch in Step 4, use the GitHub UI to initate a PR and convert it to draft mode. + +## Running CI Jobs for Database Testing + +This repository includes automated CI jobs to validate changes against a cloned database. If you want to trigger these jobs to test your changes in an isolated database environment, please follow the steps below: + +### 1. Add the Label + +Add the label `create_clone_and_run_schemachange` to your PR to trigger the CI workflow. This job does two things: + +* Creates a zero-copy clone of the database and run your proposed schema changes against it. +* Tests your schema changes on a cloned version of the development database, verifying that your updates work correctly without +affecting the real development database. After the PR is merged, the clone is automatically dropped to free up resources. + +> [!IMPORTANT] +> Your cloned database is a clone of the development database as it exists at the time of cloning. Please be mindful that **there may be +> other developers working on their own version of the development database, whose changes may not be reflected in your clone**. Keep an +> eye out for other PRs with the `create_clone_and_run_schemachange` label, and ensure that you are not performing schema changes on the +> same tables to avoid conflicts. + +> [!TIP] +> Your database will be named after your feature branch so it's easy to find on Snowsight. For example, if your feature branch is called +> `snow-123-new-feature`, your database might be called `SYNAPSE_DATA_WAREHOUSE_DEV_SNOW_123_NEW_FEATURE`. + +### 2. Perform Inspection using Snowsight + +You can go on Snowsight to perform manual inspection of your database. We recommend using a SQL worksheet to query for expected behavior +and ensure there is no row duplication in the new tables. + +> [!IMPORTANT] +> An initial clone of the development database will not incur new resource costs, **HOWEVER**, when a clone deviates from the original +> (e.g. new schema changes are applied for testing), the cloned database will begin to incur costs the longer it exists in our warehouse. +> **Please be mindful of the amount of time your PR stays open**, as cloned databases do not get dropped until a PR is merged. If your +> PR is open for >1 week, consider manually dropping your cloned database on Snowflake to avoid unnecessary cost. + +Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! \ No newline at end of file diff --git a/README.md b/README.md index 697fc06d..efe88da9 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ pip install "snowflake-connector-python[pandas]" "synapseclient[pandas]" python- ## Contributing -WIP +For contribution guidelines, please see the `CONTRIBUTING.md` file in this repository. ## Visualizing with Streamlit From 93253a210dc2acdd9862243cb7ab95d33a723dbd Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 21:55:41 -0400 Subject: [PATCH 45/81] updating contributing --- CONTRIBUTING.md | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 51ab3d0e..d98ded1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Welcome to the Snowflake repository! Please follow these guidelines to ensure a To start contributing, follow these steps to set up your local repository: -### 1. Clone the Repository: +### 1. Clone the Repository ```bash git clone https://github.com/Sage-Bionetworks/snowflake @@ -65,7 +65,7 @@ affecting the real development database. After the PR is merged, the clone is au > [!IMPORTANT] > Your cloned database is a clone of the development database as it exists at the time of cloning. Please be mindful that **there may be > other developers working on their own version of the development database, whose changes may not be reflected in your clone**. Keep an -> eye out for other PRs with the `create_clone_and_run_schemachange` label, and ensure that you are not performing schema changes on the +> eye out for other PRs with the `create_clone_and_run_schemachange` label, and ensure that you are not performing changes on the > same tables to avoid conflicts. > [!TIP] @@ -74,13 +74,16 @@ affecting the real development database. After the PR is merged, the clone is au ### 2. Perform Inspection using Snowsight -You can go on Snowsight to perform manual inspection of your database. We recommend using a SQL worksheet to query for expected behavior -and ensure there is no row duplication in the new tables. +You can go on Snowsight to perform manual inspection of the schema changes in your cloned database. We recommend using a SQL worksheet for manual quality assurance queries, e.g. ensure there is no row duplication in the new/updated tables. -> [!IMPORTANT] -> An initial clone of the development database will not incur new resource costs, **HOWEVER**, when a clone deviates from the original -> (e.g. new schema changes are applied for testing), the cloned database will begin to incur costs the longer it exists in our warehouse. -> **Please be mindful of the amount of time your PR stays open**, as cloned databases do not get dropped until a PR is merged. If your -> PR is open for >1 week, consider manually dropping your cloned database on Snowflake to avoid unnecessary cost. +### 3. Manually Drop the Cloned Database (Optional) + +There is a second job in the repository (`drop_clone`) that will drop your branch's database clone once it has been merged into `dev`. +In other words, once your cloned database is created for testing, it will remain open until your PR is closed (unless you manually drop it). + +An initial clone of the development database will not incur new resource costs, **HOWEVER**, when a clone deviates from the original +(e.g. new schema changes are applied for testing), the cloned database will begin to incur costs the longer it exists in our warehouse. +**Please be mindful of the amount of time your PR stays open**, as cloned databases do not get dropped until a PR is merged. If your +PR is open for >1 week, consider manually dropping your cloned database on Snowflake to avoid unnecessary cost. Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! \ No newline at end of file From 6e0d53739115a4b4aca071fddae84643132b7d7e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 21:59:43 -0400 Subject: [PATCH 46/81] move tip --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d98ded1c..e13c9970 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ This repository includes automated CI jobs to validate changes against a cloned Add the label `create_clone_and_run_schemachange` to your PR to trigger the CI workflow. This job does two things: -* Creates a zero-copy clone of the database and run your proposed schema changes against it. +* Creates a zero-copy clone of the database and runs your proposed schema changes against it. * Tests your schema changes on a cloned version of the development database, verifying that your updates work correctly without affecting the real development database. After the PR is merged, the clone is automatically dropped to free up resources. @@ -68,14 +68,14 @@ affecting the real development database. After the PR is merged, the clone is au > eye out for other PRs with the `create_clone_and_run_schemachange` label, and ensure that you are not performing changes on the > same tables to avoid conflicts. -> [!TIP] -> Your database will be named after your feature branch so it's easy to find on Snowsight. For example, if your feature branch is called -> `snow-123-new-feature`, your database might be called `SYNAPSE_DATA_WAREHOUSE_DEV_SNOW_123_NEW_FEATURE`. - ### 2. Perform Inspection using Snowsight You can go on Snowsight to perform manual inspection of the schema changes in your cloned database. We recommend using a SQL worksheet for manual quality assurance queries, e.g. ensure there is no row duplication in the new/updated tables. +> [!TIP] +> Your database will be named after your feature branch so it's easy to find on Snowsight. For example, if your feature branch is called +> `snow-123-new-feature`, your database might be called `SYNAPSE_DATA_WAREHOUSE_DEV_SNOW_123_NEW_FEATURE`. + ### 3. Manually Drop the Cloned Database (Optional) There is a second job in the repository (`drop_clone`) that will drop your branch's database clone once it has been merged into `dev`. From 9b187c1f1064bc7c8c7414865a7ae45b73118876 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:01:52 -0400 Subject: [PATCH 47/81] . --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e13c9970..072ad529 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,6 @@ In other words, once your cloned database is created for testing, it will remain An initial clone of the development database will not incur new resource costs, **HOWEVER**, when a clone deviates from the original (e.g. new schema changes are applied for testing), the cloned database will begin to incur costs the longer it exists in our warehouse. -**Please be mindful of the amount of time your PR stays open**, as cloned databases do not get dropped until a PR is merged. If your -PR is open for >1 week, consider manually dropping your cloned database on Snowflake to avoid unnecessary cost. +**Please be mindful of the amount of time your PR stays open**, as cloned databases do not get dropped until a PR is merged. For example, if your PR is open for >1 week, consider manually dropping your cloned database on Snowflake to avoid unnecessary cost. Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! \ No newline at end of file From 3bbd05c7452efbd394fb4ffb10b049e4e0e109f0 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:02:26 -0400 Subject: [PATCH 48/81] . --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 072ad529..25eba296 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing Guidelines -Welcome to the Snowflake repository! Please follow these guidelines to ensure a smooth development workflow and maintain the quality of our shared codebase. +Please follow these guidelines to ensure a smooth development workflow and maintain the quality of our shared codebase. ## Getting Started From 4f042c012d7fcbf35a708e736f5d19a937e53435 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:05:06 -0400 Subject: [PATCH 49/81] . --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 25eba296..db265b76 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,8 @@ # Contributing Guidelines -Please follow these guidelines to ensure a smooth development workflow and maintain the quality of our shared codebase. +Welcome, and thanks for your interest in contributing to the `snowflake` repository! :snowflake: + +By contributing, you are agreeing that we may redistribute your work under this [license](https://github.com/Sage-Bionetworks/snowflake/tree/snow-90-auto-db-clone?tab=License-1-ov-file#). ## Getting Started From d2a2452b72ac539dbc9bfeef44e142c73ffb0a7c Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:05:58 -0400 Subject: [PATCH 50/81] . --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db265b76..80642d11 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ By contributing, you are agreeing that we may redistribute your work under this ## Getting Started -To start contributing, follow these steps to set up your local repository: +To start contributing, follow these steps to set up and develop on your local repository: ### 1. Clone the Repository From ba646fa4e6e5ee046df264a8f0fbefe74344e3f0 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:09:44 -0400 Subject: [PATCH 51/81] remove redundancies from drop_clone job --- .github/workflows/test_with_clone.yaml | 27 -------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 16a47ef1..30092699 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -8,20 +8,6 @@ on: permissions: contents: read -# We define global environment variables needed to run the SnowSQL and schemachange commands -env: - SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} - SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} - SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} - SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWSQL_CLONE_ROLE: DATA_ENGINEER - SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} - SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} - SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} - CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" - STACK: ${{ vars.STACK }} - jobs: create_clone_and_run_schemachange: @@ -34,7 +20,6 @@ jobs: SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWSQL_CLONE_ROLE: DATA_ENGINEER - # We need to run schemachange on SYSADMIN for it to work SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} @@ -96,12 +81,7 @@ jobs: SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWSQL_CLONE_ROLE: DATA_ENGINEER - #SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN - SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} - SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} - SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" - STACK: ${{ vars.STACK }} steps: @@ -110,13 +90,6 @@ jobs: with: python-version: '3.10' - - name: Install python libraries - shell: bash - run: | - pip install schemachange==3.6.1 - pip install numpy==1.26.4 - pip install pandas==1.5.3 - - name: Install SnowSQL run: | curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash From 6e2403595c6743b8e7dcaccb44c8a550f7964b9a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:10:44 -0400 Subject: [PATCH 52/81] testing --- .github/workflows/test_with_clone.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 30092699..6343ddbb 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -73,7 +73,8 @@ jobs: drop_clone: runs-on: ubuntu-latest - if: github.event.pull_request.merged == true + #if: github.event.pull_request.merged == true + if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} From fc3a85d641395d05c36deb4d0c614a8b950f5135 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:13:40 -0400 Subject: [PATCH 53/81] testing done --- .github/workflows/test_with_clone.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 6343ddbb..30092699 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -73,8 +73,7 @@ jobs: drop_clone: runs-on: ubuntu-latest - #if: github.event.pull_request.merged == true - if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') + if: github.event.pull_request.merged == true environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} From cb218a5d91fff388a46cea049877b09887b4f2da Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 28 Oct 2024 22:19:41 -0400 Subject: [PATCH 54/81] final comment on time travel --- CONTRIBUTING.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80642d11..ea6804c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,4 +87,10 @@ An initial clone of the development database will not incur new resource costs, (e.g. new schema changes are applied for testing), the cloned database will begin to incur costs the longer it exists in our warehouse. **Please be mindful of the amount of time your PR stays open**, as cloned databases do not get dropped until a PR is merged. For example, if your PR is open for >1 week, consider manually dropping your cloned database on Snowflake to avoid unnecessary cost. +> [!NOTE] +> Keep in mind that after dropping your cloned database, you will still have access to it through Snowflake's "Time Travel" +> feature. Your database is retained through "Time Travel" for X amount of time before it is permanently deleted. To see +> how long your database can be access after dropping, run the following query in a SQL worksheet on Snowsight: +> `SHOW PARAMETERS IN DATABASE ;` + Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! \ No newline at end of file From 06996ab49e280d2bf82034635d1e13ae10f58dff Mon Sep 17 00:00:00 2001 From: Jenny V Medina Date: Tue, 29 Oct 2024 10:24:29 -0400 Subject: [PATCH 55/81] Update CONTRIBUTING.md --- CONTRIBUTING.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea6804c3..ed8c63c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,7 +30,7 @@ git fetch origin dev ### 3. Create a New Branch Off `dev` -Create and checkout your feature branch from the latest `dev` branch. Name it based on your feature or fix, and JIRA ticket number (if applicable). For example: +Create and checkout your feature branch from the latest `dev` branch. Name it based on the Jira ticket number and your feature/fix. For example: ```bash git checkout -b snow-123-new-feature origin/dev @@ -39,6 +39,10 @@ git checkout -b snow-123-new-feature origin/dev Your branch will now be tracking `origin/dev` which you can merge with or rebase onto should a merge conflict occur. For more guidance on how to resolve merge conflicts, [see here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/about-merge-conflicts#resolving-merge-conflicts). +> [!IMPORTANT] +> If you plan to run the automated testing described in section [Running CI Jobs for Database Testing](running-ci-jobs-for-database-testing), your branch name needs to start with `snow-`, +> otherwise the test deployment will fail. + ### 4. Push to The Remote Branch Once you've made your changes and committed them locally, push your branch to the remote repository: @@ -93,4 +97,4 @@ An initial clone of the development database will not incur new resource costs, > how long your database can be access after dropping, run the following query in a SQL worksheet on Snowsight: > `SHOW PARAMETERS IN DATABASE ;` -Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! \ No newline at end of file +Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! From ea0751839ae3ff2e36589c373c1bf48ffb98ae61 Mon Sep 17 00:00:00 2001 From: Jenny V Medina Date: Tue, 29 Oct 2024 10:25:49 -0400 Subject: [PATCH 56/81] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed8c63c6..60db75b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ Your branch will now be tracking `origin/dev` which you can merge with or rebase on how to resolve merge conflicts, [see here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/addressing-merge-conflicts/about-merge-conflicts#resolving-merge-conflicts). > [!IMPORTANT] -> If you plan to run the automated testing described in section [Running CI Jobs for Database Testing](running-ci-jobs-for-database-testing), your branch name needs to start with `snow-`, +> If you plan to run the automated testing described in section [Running CI Jobs for Database Testing](#running-ci-jobs-for-database-testing), your branch name needs to start with `snow-`, > otherwise the test deployment will fail. ### 4. Push to The Remote Branch From 9d4aaaa14b01976a2ef8fe19164b90fe5284633d Mon Sep 17 00:00:00 2001 From: Jenny V Medina Date: Tue, 29 Oct 2024 10:29:25 -0400 Subject: [PATCH 57/81] fix typo. small formatting update. --- CONTRIBUTING.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60db75b8..12cce318 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -94,7 +94,11 @@ An initial clone of the development database will not incur new resource costs, > [!NOTE] > Keep in mind that after dropping your cloned database, you will still have access to it through Snowflake's "Time Travel" > feature. Your database is retained through "Time Travel" for X amount of time before it is permanently deleted. To see -> how long your database can be access after dropping, run the following query in a SQL worksheet on Snowsight: -> `SHOW PARAMETERS IN DATABASE ;` +> how long your database can be accessed after dropping, run the following query in a SQL worksheet on Snowsight and look +> for the keyword `DATA_RETENTION_TIME_IN_DAYS`: +> +> ``` +> SHOW PARAMETERS IN DATABASE ; +> ``` Following these guidelines helps maintain a clean, efficient, and well-tested codebase. Thank you for contributing! From 03dfc5ac8ae02b867c78c68f555b9b2e67434b5e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 30 Oct 2024 09:49:55 -0400 Subject: [PATCH 58/81] only run create_clone on open prs. run drop_clone on closed and merged prs. --- .github/workflows/test_with_clone.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 30092699..9faf0611 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -12,7 +12,7 @@ jobs: create_clone_and_run_schemachange: runs-on: ubuntu-latest - if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') + if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') && github.event.pull_request.state == 'open' environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} @@ -73,7 +73,7 @@ jobs: drop_clone: runs-on: ubuntu-latest - if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true || github.event.action == 'closed' environment: dev env: SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} From ab9297a712bf260fa53f7e5dc247fc1acd85a2c8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 30 Oct 2024 10:00:30 -0400 Subject: [PATCH 59/81] improve clarity in contributing.md --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12cce318..d1f1f25a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,6 +56,9 @@ git push origin snow-123-new-feature In order to initiate automated testing you will need to work on a draft pull request (PR) on GitHub. After pushing your commits to the remote branch in Step 4, use the GitHub UI to initate a PR and convert it to draft mode. +After testing your changes against `schemachange` using the instructions in [Running CI Jobs for Database Testing](#running-ci-jobs-for-database-testing), +you can then take your PR out of draft mode by marking it as Ready for Review in the GitHub UI. + ## Running CI Jobs for Database Testing This repository includes automated CI jobs to validate changes against a cloned database. If you want to trigger these jobs to test your changes in an isolated database environment, please follow the steps below: @@ -74,6 +77,10 @@ affecting the real development database. After the PR is merged, the clone is au > eye out for other PRs with the `create_clone_and_run_schemachange` label, and ensure that you are not performing changes on the > same tables to avoid conflicts. +> [!NOTE] +> As you are developing on your branch, you may want to re-run the `schemachange` test on your updates. +> You can unlabel and relabel the PR with `create_clone_and_run_schemachange` to re-trigger the job. + ### 2. Perform Inspection using Snowsight You can go on Snowsight to perform manual inspection of the schema changes in your cloned database. We recommend using a SQL worksheet for manual quality assurance queries, e.g. ensure there is no row duplication in the new/updated tables. From 8498feb544dde50c7dbd166cc0b24a24159425d5 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 31 Oct 2024 16:36:18 -0400 Subject: [PATCH 60/81] rephrasing --- CONTRIBUTING.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d1f1f25a..6d82da7b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -72,10 +72,8 @@ Add the label `create_clone_and_run_schemachange` to your PR to trigger the CI w affecting the real development database. After the PR is merged, the clone is automatically dropped to free up resources. > [!IMPORTANT] -> Your cloned database is a clone of the development database as it exists at the time of cloning. Please be mindful that **there may be -> other developers working on their own version of the development database, whose changes may not be reflected in your clone**. Keep an -> eye out for other PRs with the `create_clone_and_run_schemachange` label, and ensure that you are not performing changes on the -> same tables to avoid conflicts. +> Your cloned database is a clone of the development database as it exists at the time of cloning. Please be mindful that +> **there may have been changes made to the development database since your last clone**. > [!NOTE] > As you are developing on your branch, you may want to re-run the `schemachange` test on your updates. From ed645f352fe553f636ac7a95f01f42681a6633e8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 13:27:57 -0500 Subject: [PATCH 61/81] use snowflake cli --- .github/workflows/test_with_clone.yaml | 74 ++++++++++++++++---------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 9faf0611..9c00bdbd 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -15,12 +15,12 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'create_clone_and_run_schemachange') && github.event.pull_request.state == 'open' environment: dev env: - SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} - SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} - SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} - SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWSQL_CLONE_ROLE: DATA_ENGINEER - SNOWSQL_SCHEMACHANGE_ROLE: SYSADMIN + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWSQL_PWD }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWFLAKE_CLONE_ROLE: DATA_ENGINEER + SNOWFLAKE_SCHEMACHANGE_ROLE: SYSADMIN SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} @@ -41,14 +41,22 @@ jobs: pip install numpy==1.26.4 pip install pandas==1.5.3 - - name: Install SnowSQL + # - name: Install SnowSQL + # run: | + # curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash + # SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + + # - name: Verify SnowSQL installation + # run: ~/bin/snowsql -v + + - name: Install Snowflake CLI + uses: Snowflake-Labs/snowflake-cli-action@v1.5 + + - name: Verify Snowflake CLI installation run: | - curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash - SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + snow --version + snow connection test - - name: Verify SnowSQL installation - run: ~/bin/snowsql -v - - name: Sanitize Clone Name run: | CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" @@ -58,17 +66,17 @@ jobs: - name: Zero-copy clone the database shell: bash run: | - ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" + snow sql -r $SNOWFLAKE_CLONE_ROLE -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" - name: Run schemachange on the clone shell: bash run: | schemachange \ -f synapse_data_warehouse \ - -a $SNOWSQL_ACCOUNT \ - -u $SNOWSQL_USER \ - -r $SNOWSQL_SCHEMACHANGE_ROLE \ - -w $SNOWSQL_WAREHOUSE \ + -a $SNOWFLAKE_ACCOUNT \ + -u $SNOWFLAKE_USER \ + -r $SNOWFLAKE_SCHEMACHANGE_ROLE \ + -w $SNOWFLAKE_WAREHOUSE \ --config-folder synapse_data_warehouse drop_clone: @@ -76,11 +84,11 @@ jobs: if: github.event.pull_request.merged == true || github.event.action == 'closed' environment: dev env: - SNOWSQL_PWD: ${{ secrets.SNOWSQL_PWD }} - SNOWSQL_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} - SNOWSQL_USER: ${{ secrets.SNOWSQL_USER }} - SNOWSQL_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} - SNOWSQL_CLONE_ROLE: DATA_ENGINEER + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWSQL_PWD }} + SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWSQL_ACCOUNT }} + SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }} + SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} + SNOWFLAKE_CLONE_ROLE: DATA_ENGINEER CLONE_NAME: "${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }}_${{ github.head_ref }}" steps: @@ -90,14 +98,22 @@ jobs: with: python-version: '3.10' - - name: Install SnowSQL + # - name: Install SnowSQL + # run: | + # curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash + # SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + + # - name: Verify SnowSQL installation + # run: ~/bin/snowsql -v + + - name: Install Snowflake CLI + uses: Snowflake-Labs/snowflake-cli-action@v1.5 + + - name: Verify Snowflake CLI installation run: | - curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash - SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash + snow --version + snow connection test - - name: Verify SnowSQL installation - run: ~/bin/snowsql -v - - name: Sanitize Clone Name run: | CLONE_NAME_SANITIZED="${CLONE_NAME//[^a-zA-Z0-9_]/_}" @@ -108,4 +124,4 @@ jobs: - name: Drop the clone shell: bash run: | - ~/bin/snowsql -r $SNOWSQL_CLONE_ROLE -q "DROP DATABASE IF EXISTS $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE;" \ No newline at end of file + snow sql -r $SNOWFLAKE_CLONE_ROLE -q "DROP DATABASE IF EXISTS $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE;" \ No newline at end of file From ced4d2c589500fe51f97b73a261522f4fdc2237a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 13:36:37 -0500 Subject: [PATCH 62/81] snowflake connection --- .github/workflows/test_with_clone.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 9c00bdbd..3e001264 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -109,7 +109,17 @@ jobs: - name: Install Snowflake CLI uses: Snowflake-Labs/snowflake-cli-action@v1.5 - - name: Verify Snowflake CLI installation + - name: Set up Snowflake CLI connection + run: | + snow connection create --name default \ + --account-name $SNOWFLAKE_ACCOUNT \ + --username $SNOWFLAKE_USER \ + --password $SNOWFLAKE_PASSWORD \ + --role $SNOWFLAKE_CLONE_ROLE \ + --warehouse $SNOWFLAKE_WAREHOUSE \ + --database $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG + + - name: Verify Snowflake CLI installation and connection run: | snow --version snow connection test From 3a90e924451e4fe87afcefba8b0880b524c2636f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 14:01:59 -0500 Subject: [PATCH 63/81] . --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 3e001264..68678159 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -45,7 +45,7 @@ jobs: # run: | # curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash # SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - + # - name: Verify SnowSQL installation # run: ~/bin/snowsql -v From a85a5ca7a76bae9935f659755ccb317c474cbb5d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 14:06:18 -0500 Subject: [PATCH 64/81] lol --- .github/workflows/test_with_clone.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 68678159..f0f4346e6 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -52,6 +52,16 @@ jobs: - name: Install Snowflake CLI uses: Snowflake-Labs/snowflake-cli-action@v1.5 + - name: Set up Snowflake CLI connection + run: | + snow connection create --name default \ + --account-name $SNOWFLAKE_ACCOUNT \ + --username $SNOWFLAKE_USER \ + --password $SNOWFLAKE_PASSWORD \ + --role $SNOWFLAKE_CLONE_ROLE \ + --warehouse $SNOWFLAKE_WAREHOUSE \ + --database $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG + - name: Verify Snowflake CLI installation run: | snow --version From 8e78bd8408215f90b66b51e158265c2750037f67 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 14:26:47 -0500 Subject: [PATCH 65/81] . --- .github/workflows/test_with_clone.yaml | 29 ++++++++++++-------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index f0f4346e6..bb41d242 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -41,26 +41,23 @@ jobs: pip install numpy==1.26.4 pip install pandas==1.5.3 - # - name: Install SnowSQL - # run: | - # curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash - # SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - - # - name: Verify SnowSQL installation - # run: ~/bin/snowsql -v - - name: Install Snowflake CLI uses: Snowflake-Labs/snowflake-cli-action@v1.5 - - name: Set up Snowflake CLI connection + - name: Configure Snowflake connection run: | - snow connection create --name default \ - --account-name $SNOWFLAKE_ACCOUNT \ - --username $SNOWFLAKE_USER \ - --password $SNOWFLAKE_PASSWORD \ - --role $SNOWFLAKE_CLONE_ROLE \ - --warehouse $SNOWFLAKE_WAREHOUSE \ - --database $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG + # Create temporary files for config.toml and our private key + config_file=$(mktemp) + + # Write to config.toml file + echo 'default_connection_name = "dpe"' >> $config_file + echo '[connections.dpe]' >> $config_file + echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file + echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file + echo "role = \"${SNOWFLAKE_CLONE_ROLE}\"" >> $config_file + echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file + echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file + echo 'authenticator = "SNOWFLAKE"' >> $config_file - name: Verify Snowflake CLI installation run: | From 294244f639da4e598bd7d11baf55a8a0cb5ce28f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 14:30:02 -0500 Subject: [PATCH 66/81] . --- .github/workflows/test_with_clone.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index bb41d242..1a35ff16 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -41,9 +41,6 @@ jobs: pip install numpy==1.26.4 pip install pandas==1.5.3 - - name: Install Snowflake CLI - uses: Snowflake-Labs/snowflake-cli-action@v1.5 - - name: Configure Snowflake connection run: | # Create temporary files for config.toml and our private key @@ -59,7 +56,15 @@ jobs: echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file echo 'authenticator = "SNOWFLAKE"' >> $config_file - - name: Verify Snowflake CLI installation + # Write config.toml path to global environment + echo "SNOWFLAKE_CONFIG_PATH=$config_file" >> $GITHUB_ENV + + - name: Install Snowflake CLI + uses: Snowflake-Labs/snowflake-cli-action@v1.5 + with: + default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH }} + + - name: Verify Snowflake CLI installation and connection run: | snow --version snow connection test From f75fb496093769114edcafee4637255269507a48 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 4 Nov 2024 15:29:13 -0500 Subject: [PATCH 67/81] we no longer need to auth via the query command --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 1a35ff16..d9882476 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -78,7 +78,7 @@ jobs: - name: Zero-copy clone the database shell: bash run: | - snow sql -r $SNOWFLAKE_CLONE_ROLE -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" + snow sql -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" - name: Run schemachange on the clone shell: bash From 65cfebbea9c6921deef672f9ce07cf6d510b846d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 09:16:55 -0500 Subject: [PATCH 68/81] run for every commit when the PR is labeled --- .github/workflows/test_with_clone.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index d9882476..0f63e019 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -3,7 +3,8 @@ name: Test Changes with Cloned DB on: pull_request: - types: [ labeled, closed ] + types: [ labeled, synchronize, closed ] + push: permissions: contents: read From 9d8fde06b7ff41a7b7dfee38aa84cf031d086a2e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 09:27:59 -0500 Subject: [PATCH 69/81] V script to make dummy table --- .../synapse/tables/V2.20.1__my_table.sql | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql diff --git a/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql b/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql new file mode 100644 index 00000000..8396220e --- /dev/null +++ b/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql @@ -0,0 +1,17 @@ +USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP + +-- Create the dummy table with example columns +CREATE TABLE IF NOT EXISTS my_table ( + id INT, + name STRING, + created_at TIMESTAMP, + value FLOAT +); + +-- Insert arbitrary rows into the dummy table +INSERT INTO my_table (id, name, created_at, value) VALUES + (1, 'Alpha', CURRENT_TIMESTAMP, 10.5), + (2, 'Beta', CURRENT_TIMESTAMP, 20.0), + (3, 'Gamma', CURRENT_TIMESTAMP, 30.75), + (4, 'Delta', CURRENT_TIMESTAMP, 40.1), + (5, 'Epsilon', CURRENT_TIMESTAMP, 50.9); \ No newline at end of file From 6f2052a7ba4a2bda6e1312c1df7044a91adc0e01 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 09:28:26 -0500 Subject: [PATCH 70/81] quick change --- synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql b/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql index 8396220e..831fd12c 100644 --- a/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql +++ b/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS my_table ( -- Insert arbitrary rows into the dummy table INSERT INTO my_table (id, name, created_at, value) VALUES - (1, 'Alpha', CURRENT_TIMESTAMP, 10.5), + (1, 'Alpha', CURRENT_TIMESTAMP, 1000.5), (2, 'Beta', CURRENT_TIMESTAMP, 20.0), (3, 'Gamma', CURRENT_TIMESTAMP, 30.75), (4, 'Delta', CURRENT_TIMESTAMP, 40.1), From 55cbfd848dd85024a1521fe50ab4667f885f8fee Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 09:38:48 -0500 Subject: [PATCH 71/81] rename V to see if that fixes the problem --- .../tables/{V2.20.1__my_table.sql => v2.20.2__my_table.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse_data_warehouse/synapse/tables/{V2.20.1__my_table.sql => v2.20.2__my_table.sql} (100%) diff --git a/synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql b/synapse_data_warehouse/synapse/tables/v2.20.2__my_table.sql similarity index 100% rename from synapse_data_warehouse/synapse/tables/V2.20.1__my_table.sql rename to synapse_data_warehouse/synapse/tables/v2.20.2__my_table.sql From 9a622caad089a4e6406f1c5564512c858aad9def Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 09:45:22 -0500 Subject: [PATCH 72/81] change to R script --- .../synapse/tables/{v2.20.2__my_table.sql => R__my_table.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename synapse_data_warehouse/synapse/tables/{v2.20.2__my_table.sql => R__my_table.sql} (100%) diff --git a/synapse_data_warehouse/synapse/tables/v2.20.2__my_table.sql b/synapse_data_warehouse/synapse/tables/R__my_table.sql similarity index 100% rename from synapse_data_warehouse/synapse/tables/v2.20.2__my_table.sql rename to synapse_data_warehouse/synapse/tables/R__my_table.sql From 2c9bab2d9e58b6db207ab0dea727a985f9d36b43 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 10:01:15 -0500 Subject: [PATCH 73/81] rename table. update drop_clone job. --- .github/workflows/test_with_clone.yaml | 38 ++++++++++--------- .../synapse/tables/R__my_table.sql | 2 +- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 0f63e019..6a4f3f3d 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -111,26 +111,28 @@ jobs: with: python-version: '3.10' - # - name: Install SnowSQL - # run: | - # curl -O https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-1.2.9-linux_x86_64.bash - # SNOWSQL_DEST=~/bin SNOWSQL_LOGIN_SHELL=~/.profile bash snowsql-1.2.9-linux_x86_64.bash - - # - name: Verify SnowSQL installation - # run: ~/bin/snowsql -v - + - name: Configure Snowflake connection + run: | + # Create temporary files for config.toml and our private key + config_file=$(mktemp) + + # Write to config.toml file + echo 'default_connection_name = "dpe"' >> $config_file + echo '[connections.dpe]' >> $config_file + echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file + echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file + echo "role = \"${SNOWFLAKE_CLONE_ROLE}\"" >> $config_file + echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file + echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file + echo 'authenticator = "SNOWFLAKE"' >> $config_file + + # Write config.toml path to global environment + echo "SNOWFLAKE_CONFIG_PATH=$config_file" >> $GITHUB_ENV + - name: Install Snowflake CLI uses: Snowflake-Labs/snowflake-cli-action@v1.5 - - - name: Set up Snowflake CLI connection - run: | - snow connection create --name default \ - --account-name $SNOWFLAKE_ACCOUNT \ - --username $SNOWFLAKE_USER \ - --password $SNOWFLAKE_PASSWORD \ - --role $SNOWFLAKE_CLONE_ROLE \ - --warehouse $SNOWFLAKE_WAREHOUSE \ - --database $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG + with: + default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH }} - name: Verify Snowflake CLI installation and connection run: | diff --git a/synapse_data_warehouse/synapse/tables/R__my_table.sql b/synapse_data_warehouse/synapse/tables/R__my_table.sql index 831fd12c..4b273910 100644 --- a/synapse_data_warehouse/synapse/tables/R__my_table.sql +++ b/synapse_data_warehouse/synapse/tables/R__my_table.sql @@ -1,7 +1,7 @@ USE SCHEMA {{database_name}}.synapse; --noqa: JJ01,PRS,TMP -- Create the dummy table with example columns -CREATE TABLE IF NOT EXISTS my_table ( +CREATE TABLE IF NOT EXISTS my_table2 ( id INT, name STRING, created_at TIMESTAMP, From 912b97467a1a7ffd7725f2ab1eaafca626bd9f90 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 10:04:28 -0500 Subject: [PATCH 74/81] . --- synapse_data_warehouse/synapse/tables/R__my_table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse_data_warehouse/synapse/tables/R__my_table.sql b/synapse_data_warehouse/synapse/tables/R__my_table.sql index 4b273910..a17c93f5 100644 --- a/synapse_data_warehouse/synapse/tables/R__my_table.sql +++ b/synapse_data_warehouse/synapse/tables/R__my_table.sql @@ -9,7 +9,7 @@ CREATE TABLE IF NOT EXISTS my_table2 ( ); -- Insert arbitrary rows into the dummy table -INSERT INTO my_table (id, name, created_at, value) VALUES +INSERT INTO my_table2 (id, name, created_at, value) VALUES (1, 'Alpha', CURRENT_TIMESTAMP, 1000.5), (2, 'Beta', CURRENT_TIMESTAMP, 20.0), (3, 'Gamma', CURRENT_TIMESTAMP, 30.75), From d0880d8f9042354f1c65da6ac549e8f483f09749 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 10:08:16 -0500 Subject: [PATCH 75/81] try running schemachange as data_engineer --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 6a4f3f3d..c15a2c27 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -21,7 +21,7 @@ jobs: SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }} SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWFLAKE_CLONE_ROLE: DATA_ENGINEER - SNOWFLAKE_SCHEMACHANGE_ROLE: SYSADMIN + SNOWFLAKE_SCHEMACHANGE_ROLE: DATA_ENGINEER SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} From c6ba8d56879a772fe016110a984b460e07c3e38a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 5 Nov 2024 10:21:03 -0500 Subject: [PATCH 76/81] . --- .github/workflows/test_with_clone.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index c15a2c27..6a4f3f3d 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -21,7 +21,7 @@ jobs: SNOWFLAKE_USER: ${{ secrets.SNOWSQL_USER }} SNOWFLAKE_WAREHOUSE: ${{ secrets.SNOWSQL_WAREHOUSE }} SNOWFLAKE_CLONE_ROLE: DATA_ENGINEER - SNOWFLAKE_SCHEMACHANGE_ROLE: DATA_ENGINEER + SNOWFLAKE_SCHEMACHANGE_ROLE: SYSADMIN SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG: ${{ vars.SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE }} SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_STORAGE_INTEGRATION }} SNOWFLAKE_SYNAPSE_STAGE_URL: ${{ vars.SNOWFLAKE_SYNAPSE_STAGE_URL }} From 8b50065fcdb60e4ad4e0ab2180372489ecaa9828 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 6 Nov 2024 14:26:24 -0500 Subject: [PATCH 77/81] grant privileges to dpe_engineer --- .github/workflows/test_with_clone.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 6a4f3f3d..1d377d8f 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -52,7 +52,7 @@ jobs: echo '[connections.dpe]' >> $config_file echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file - echo "role = \"${SNOWFLAKE_CLONE_ROLE}\"" >> $config_file + echo "role = \"${SNOWFLAKE_SCHEMACHANGE_ROLE}\"" >> $config_file echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file echo 'authenticator = "SNOWFLAKE"' >> $config_file @@ -79,7 +79,13 @@ jobs: - name: Zero-copy clone the database shell: bash run: | - snow sql -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" + snow sql -q "USE ROLE DPE_ENGINEER; CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" + + - name: Grant permissions to DPE_ENGINEER on cloned database schemas of interest + shell: bash + run: | + snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE TO ROLE DPE_ENGINEER;" + snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE_RAW TO ROLE DPE_ENGINEER;" - name: Run schemachange on the clone shell: bash @@ -88,7 +94,7 @@ jobs: -f synapse_data_warehouse \ -a $SNOWFLAKE_ACCOUNT \ -u $SNOWFLAKE_USER \ - -r $SNOWFLAKE_SCHEMACHANGE_ROLE \ + -r DPE_ENGINEER \ -w $SNOWFLAKE_WAREHOUSE \ --config-folder synapse_data_warehouse From 00c2a3c013cf17bac72dcb4afe89eb7561cad164 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 6 Nov 2024 14:55:21 -0500 Subject: [PATCH 78/81] attempt at using dpe_engineer for schematic again --- .github/workflows/test_with_clone.yaml | 62 +++++++++++++++----------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 1d377d8f..a29cf957 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -42,33 +42,44 @@ jobs: pip install numpy==1.26.4 pip install pandas==1.5.3 - - name: Configure Snowflake connection + - name: Configure Snowflake connections run: | - # Create temporary files for config.toml and our private key - config_file=$(mktemp) - - # Write to config.toml file - echo 'default_connection_name = "dpe"' >> $config_file - echo '[connections.dpe]' >> $config_file - echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file - echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file - echo "role = \"${SNOWFLAKE_SCHEMACHANGE_ROLE}\"" >> $config_file - echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file - echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file - echo 'authenticator = "SNOWFLAKE"' >> $config_file - - # Write config.toml path to global environment - echo "SNOWFLAKE_CONFIG_PATH=$config_file" >> $GITHUB_ENV - - - name: Install Snowflake CLI + # Config file for DPE_ENGINEER + config_file_dpe=$(mktemp) + echo 'default_connection_name = "dpe"' >> $config_file_dpe + echo '[connections.dpe]' >> $config_file_dpe + echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file_dpe + echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file_dpe + echo "role = \"${SNOWFLAKE_CLONE_ROLE}\"" >> $config_file_dpe + echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file_dpe + echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file_dpe + echo 'authenticator = "SNOWFLAKE"' >> $config_file_dpe + + # Config file for SYSADMIN + config_file_sysadmin=$(mktemp) + echo 'default_connection_name = "sysadmin"' >> $config_file_sysadmin + echo '[connections.sysadmin]' >> $config_file_sysadmin + echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file_sysadmin + echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file_sysadmin + echo "role = \"${SNOWFLAKE_SCHEMACHANGE_ROLE}\"" >> $config_file_sysadmin + echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file_sysadmin + echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file_sysadmin + echo 'authenticator = "SNOWFLAKE"' >> $config_file_sysadmin + + # Write config paths to environment + echo "SNOWFLAKE_CONFIG_PATH_DPE=$config_file_dpe" >> $GITHUB_ENV + echo "SNOWFLAKE_CONFIG_PATH_SYSADMIN=$config_file_sysadmin" >> $GITHUB_ENV + + - name: Install Snowflake CLI with DPE_ENGINEER config uses: Snowflake-Labs/snowflake-cli-action@v1.5 with: - default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH }} + default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH_DPE }} - - name: Verify Snowflake CLI installation and connection + - name: Verify Snowflake CLI installation and connections run: | snow --version - snow connection test + snow connection test -c sysadmin + snow connection test -c dpe - name: Sanitize Clone Name run: | @@ -79,22 +90,23 @@ jobs: - name: Zero-copy clone the database shell: bash run: | - snow sql -q "USE ROLE DPE_ENGINEER; CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" + snow sql -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" - - name: Grant permissions to DPE_ENGINEER on cloned database schemas of interest + - name: Grant permissions to DPE_ENGINEER on cloned database shell: bash run: | + snow connection set-default sysadmin snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE TO ROLE DPE_ENGINEER;" snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE_RAW TO ROLE DPE_ENGINEER;" - - name: Run schemachange on the clone + - name: Run schemachange on the clone as DPE_ENGINEER shell: bash run: | schemachange \ -f synapse_data_warehouse \ -a $SNOWFLAKE_ACCOUNT \ -u $SNOWFLAKE_USER \ - -r DPE_ENGINEER \ + -r $SNOWFLAKE_CLONE_ROLE \ -w $SNOWFLAKE_WAREHOUSE \ --config-folder synapse_data_warehouse From 5babc721576ec4a97d3d4d8607116856a3d1ac67 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 6 Nov 2024 15:00:13 -0500 Subject: [PATCH 79/81] put both configs in the same file --- .github/workflows/test_with_clone.yaml | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index a29cf957..4167bdd3 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -56,19 +56,16 @@ jobs: echo 'authenticator = "SNOWFLAKE"' >> $config_file_dpe # Config file for SYSADMIN - config_file_sysadmin=$(mktemp) - echo 'default_connection_name = "sysadmin"' >> $config_file_sysadmin - echo '[connections.sysadmin]' >> $config_file_sysadmin - echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file_sysadmin - echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file_sysadmin - echo "role = \"${SNOWFLAKE_SCHEMACHANGE_ROLE}\"" >> $config_file_sysadmin - echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file_sysadmin - echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file_sysadmin - echo 'authenticator = "SNOWFLAKE"' >> $config_file_sysadmin + echo '[connections.sysadmin]' >> $config_file_dpe + echo "account = \"${SNOWFLAKE_ACCOUNT}\"" >> $config_file_dpe + echo "user = \"${SNOWFLAKE_USER}\"" >> $config_file_dpe + echo "role = \"${SNOWFLAKE_SCHEMACHANGE_ROLE}\"" >> $config_file_dpe + echo "password = \"${SNOWFLAKE_PASSWORD}\"" >> $config_file_dpe + echo "warehouse = \"${SNOWFLAKE_WAREHOUSE}\"" >> $config_file_dpe + echo 'authenticator = "SNOWFLAKE"' >> $config_file_dpe # Write config paths to environment echo "SNOWFLAKE_CONFIG_PATH_DPE=$config_file_dpe" >> $GITHUB_ENV - echo "SNOWFLAKE_CONFIG_PATH_SYSADMIN=$config_file_sysadmin" >> $GITHUB_ENV - name: Install Snowflake CLI with DPE_ENGINEER config uses: Snowflake-Labs/snowflake-cli-action@v1.5 From 97d0b7b874ceda0951194a137fa384ae868fc1f8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 7 Nov 2024 09:50:55 -0500 Subject: [PATCH 80/81] typo --- .github/workflows/test_with_clone.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 4167bdd3..7a3bde12 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -44,7 +44,7 @@ jobs: - name: Configure Snowflake connections run: | - # Config file for DPE_ENGINEER + # Config file for DATA_ENGINEER config_file_dpe=$(mktemp) echo 'default_connection_name = "dpe"' >> $config_file_dpe echo '[connections.dpe]' >> $config_file_dpe @@ -67,7 +67,7 @@ jobs: # Write config paths to environment echo "SNOWFLAKE_CONFIG_PATH_DPE=$config_file_dpe" >> $GITHUB_ENV - - name: Install Snowflake CLI with DPE_ENGINEER config + - name: Install Snowflake CLI with DATA_ENGINEER config uses: Snowflake-Labs/snowflake-cli-action@v1.5 with: default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH_DPE }} @@ -89,14 +89,14 @@ jobs: run: | snow sql -q "CREATE OR REPLACE DATABASE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE CLONE $SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE_ORIG;" - - name: Grant permissions to DPE_ENGINEER on cloned database + - name: Grant permissions to DATA_ENGINEER on cloned database shell: bash run: | snow connection set-default sysadmin - snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE TO ROLE DPE_ENGINEER;" - snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE_RAW TO ROLE DPE_ENGINEER;" + snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE TO ROLE DATA_ENGINEER;" + snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE_RAW TO ROLE DATA_ENGINEER;" - - name: Run schemachange on the clone as DPE_ENGINEER + - name: Run schemachange on the clone as DATA_ENGINEER shell: bash run: | schemachange \ From 517525fc3640069f28ee4ba7d6a7b1a301c36976 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 7 Nov 2024 10:28:19 -0500 Subject: [PATCH 81/81] grant ownership to CLONE database --- .github/workflows/test_with_clone.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_with_clone.yaml b/.github/workflows/test_with_clone.yaml index 7a3bde12..ef3b2551 100644 --- a/.github/workflows/test_with_clone.yaml +++ b/.github/workflows/test_with_clone.yaml @@ -93,8 +93,8 @@ jobs: shell: bash run: | snow connection set-default sysadmin - snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE TO ROLE DATA_ENGINEER;" - snow sql -q "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE_RAW TO ROLE DATA_ENGINEER;" + snow sql -q "GRANT OWNERSHIP ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE_RAW TO ROLE DATA_ENGINEER REVOKE CURRENT GRANTS;" + snow sql -q "GRANT OWNERSHIP ON ALL TABLES IN SCHEMA ${SNOWFLAKE_SYNAPSE_DATA_WAREHOUSE_DATABASE}.SYNAPSE TO ROLE DATA_ENGINEER REVOKE CURRENT GRANTS;" - name: Run schemachange on the clone as DATA_ENGINEER shell: bash