GMT-109: Snowflake writer supporting GCP

.env.dist

@@ -1,4 +1,3 @@
-STORAGE_API_TOKEN=
 KBC_URL=
 KBC_RUNID=
 DB_HOST=

.github/workflows/push.yml

@@ -1,16 +1,33 @@
 name: 'GitHub Actions'
-'on':
-  - push
+on: [ push ]
 concurrency: 'ci-${{ github.ref }}'
 env:
-  APP_IMAGE: keboola-component
+  # Name of the image
+  APP_IMAGE: keboola/wr-db-snowflake-gcs # must be same in docker-compose.yml
+
+  # Developer portal login
   KBC_DEVELOPERPORTAL_VENDOR: keboola
-  KBC_DEVELOPERPORTAL_APP: keboola.db-writer-snowflake-gcs
+  KBC_DEVELOPERPORTAL_APP: keboola.wr-db-snowflake-gcs
   KBC_DEVELOPERPORTAL_USERNAME: keboola+wr_db_snowflake_gcs
-  KBC_DEVELOPERPORTAL_PASSWORD: '${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}'
-  DOCKERHUB_USER: ''
-  DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}'
-  KBC_STORAGE_TOKEN: '${{ secrets.KBC_STORAGE_TOKEN }}'
+  KBC_DEVELOPERPORTAL_PASSWORD: ${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}
+
+  # DockerHub login
+  DOCKERHUB_USER: keboolabot
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
+  # Test DB
+  DB_HOST: kebooladev.snowflakecomputing.com
+  DB_PORT: 443
+  DB_USER: snowflake_writer_gcs
+  DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
+  DB_DATABASE: snowflake_writer_gcs
+  DB_SCHEMA: snowflake_writer_gcs
+  DB_WAREHOUSE: snowflake_writer_gcs
+
+  # Testing staging storage projects
+  KBC_RUNID: 123456
+  KBC_STORAGE_TOKEN: ''
+  KBC_URL: 'https://connection.keboola.com/'
   KBC_TEST_PROJECT_URL: ''
   KBC_TEST_PROJECT_CONFIGS: ''
 jobs:
@@ -21,7 +38,7 @@ jobs:
       is_semantic_tag: '${{ steps.tag.outputs.is_semantic_tag }}'
     steps:
       - name: 'Check out the repo'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Print Docker version'
         run: 'docker -v'
       - name: 'Docker login'
@@ -40,31 +57,33 @@ jobs:
       - name: 'Push image to ECR'
         uses: keboola/action-push-to-ecr@master
         with:
-          vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
-          app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-          username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
-          password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
-          tag: '${{ steps.tag.outputs.app_image_tag }}'
-          push_latest: '${{ steps.tag.outputs.is_semantic_tag }}'
-          source_image: '${{ env.APP_IMAGE}}'
+          vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
+          app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
+          password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
+          tag: ${{ steps.tag.outputs.app_image_tag }}
+          push_latest: ${{ steps.tag.outputs.is_semantic_tag }}
+          source_image: ${{ env.APP_IMAGE}}
   tests:
     needs: build
     runs-on: ubuntu-latest
     steps:
       - name: 'Check out the repo'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Pull image from ECR'
         uses: keboola/action-pull-from-ecr@master
         with:
-          vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
-          app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-          username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
-          password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
-          tag: '${{ needs.build.outputs.app_image_tag }}'
-          target_image: '${{ env.APP_IMAGE}}'
+          vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
+          app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
+          password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
+          tag: ${{ needs.build.outputs.app_image_tag }}
+          target_image: ${{ env.APP_IMAGE}}
           tag_as_latest: true
       - name: 'Run tests'
-        run: 'docker run ${{env.APP_IMAGE}} composer ci'
+        run: |
+          docker-compose run --rm \
+          app composer ci
   tests-in-kbc:
     needs: build
     runs-on: ubuntu-latest
@@ -73,10 +92,10 @@ jobs:
         if: 'env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS'
         uses: keboola/action-run-configs-parallel@master
         with:
-          token: '${{ env.KBC_STORAGE_TOKEN }}'
-          componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-          tag: '${{ needs.build.outputs.app_image_tag }}'
-          configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}'
+          token: ${{ env.KBC_STORAGE_TOKEN }}
+          componentId: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          tag: ${{ needs.build.outputs.app_image_tag }}
+          configs: ${{ env.KBC_TEST_PROJECT_CONFIGS }}
   deploy:
     needs:
       - build
@@ -88,8 +107,8 @@ jobs:
       - name: 'Set tag in the Developer Portal'
         uses: keboola/action-set-tag-developer-portal@master
         with:
-          vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
-          app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-          username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
-          password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
-          tag: '${{ needs.build.outputs.app_image_tag }}'
+          vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
+          app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
+          username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
+          password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
+          tag: ${{ needs.build.outputs.app_image_tag }}

README.md

@@ -9,97 +9,104 @@ Writes data to Snowflake Database.
 ## Example configuration
 
 ```json
-    {
-      "db": {        
-        "host": "HOST",
-        "port": "PORT",
-        "database": "DATABASE",
-        "user": "USERNAME",
-        "password": "PASSWORD",
-        "schema": "SCHEMA",
-        "warehouse": "WAREHOUSE",
-        "ssh": {
-          "enabled": true,
-          "keys": {
-            "private": "ENCRYPTED_PRIVATE_SSH_KEY",
-            "public": "PUBLIC_SSH_KEY"
-          },
-          "sshHost": "PROXY_HOSTNAME"
-        }
+{
+  "parameters": {
+    "db": {
+      "host": "HOST",
+      "port": "PORT",
+      "database": "DATABASE",
+      "user": "USERNAME",
+      "password": "PASSWORD",
+      "schema": "SCHEMA",
+      "warehouse": "WAREHOUSE",
+      "ssh": {
+        "enabled": true,
+        "keys": {
+          "private": "ENCRYPTED_PRIVATE_SSH_KEY",
+          "public": "PUBLIC_SSH_KEY"
+        },
+        "sshHost": "PROXY_HOSTNAME"
+      }
+    },
+    "tableId": "simple",
+    "dbName": "simple",
+    "export": true,
+    "incremental": false,
+    "primaryKey": ["id"],
+    "items": [
+      {
+        "name": "id",
+        "dbName": "id",
+        "type": "int",
+        "size": null,
+        "nullable": null,
+        "default": null
+      },
+      {
+        "name": "name",
+        "dbName": "name",
+        "type": "varchar",
+        "size": 255,
+        "nullable": null,
+        "default": null
       },
+      {
+        "name": "glasses",
+        "dbName": "glasses",
+        "type": "varchar",
+        "size": 255,
+        "nullable": null,
+        "default": null
+      }
+    ]
+  },
+  "storage": {
+    "input": {
       "tables": [
         {
-          "tableId": "simple",
-          "dbName": "simple",
-          "export": true, 
-          "incremental": true,
-          "primaryKey": ["id"],
-          "items": [
-            {
-              "name": "id",
-              "dbName": "id",
-              "type": "int",
-              "size": null,
-              "nullable": null,
-              "default": null
-            },
-            {
-              "name": "name",
-              "dbName": "name",
-              "type": "nvarchar",
-              "size": 255,
-              "nullable": null,
-              "default": null
-            },
-            {
-              "name": "glasses",
-              "dbName": "glasses",
-              "type": "nvarchar",
-              "size": 255,
-              "nullable": null,
-              "default": null
-            }
-          ]                                
+          "source": "simple",
+          "destination": "simple.csv"
         }
       ]
     }
+  }
+}
 ```
 
 ## Development
 
 Required snowflake resource for writer:
 ```sql
-CREATE DATABASE "snowflake_writer";
-USE DATABASE "snowflake_writer";
-CREATE TRANSIENT SCHEMA "snowflake_writer";
-CREATE WAREHOUSE "snowflake_writer" WITH 
+CREATE DATABASE "snowflake_writer_gcs";
+USE DATABASE "snowflake_writer_gcs";
+CREATE TRANSIENT SCHEMA "snowflake_writer_gcs";
+CREATE WAREHOUSE "snowflake_writer_gcs" WITH 
   WAREHOUSE_SIZE = 'XSMALL' 
   WAREHOUSE_TYPE = 'STANDARD' 
   AUTO_SUSPEND = 900 
   AUTO_RESUME = TRUE;
-CREATE ROLE "snowflake_writer";
-GRANT USAGE ON WAREHOUSE "snowflake_writer" TO ROLE "snowflake_writer";
-GRANT USAGE ON DATABASE "snowflake_writer" TO ROLE "snowflake_writer";
-GRANT ALL ON SCHEMA "snowflake_writer" TO ROLE "snowflake_writer";
-GRANT ALL ON FUTURE TABLES IN SCHEMA "snowflake_writer" TO ROLE "snowflake_writer";
-GRANT ALL ON FUTURE VIEWS IN SCHEMA "snowflake_writer" TO ROLE "snowflake_writer";
-CREATE USER "snowflake_writer" 
+CREATE ROLE "snowflake_writer_gcs";
+GRANT USAGE ON WAREHOUSE "snowflake_writer_gcs" TO ROLE "snowflake_writer_gcs";
+GRANT USAGE ON DATABASE "snowflake_writer_gcs" TO ROLE "snowflake_writer_gcs";
+GRANT ALL ON SCHEMA "snowflake_writer_gcs" TO ROLE "snowflake_writer_gcs";
+GRANT ALL ON FUTURE TABLES IN SCHEMA "snowflake_writer_gcs" TO ROLE "snowflake_writer_gcs";
+GRANT ALL ON FUTURE VIEWS IN SCHEMA "snowflake_writer_gcs" TO ROLE "snowflake_writer_gcs";
+CREATE USER "snowflake_writer_gcs" 
   PASSWORD = 'password' 
-  DEFAULT_ROLE = "snowflake_writer" 
-  DEFAULT_WAREHOUSE = "snowflake_writer" 
-  DEFAULT_NAMESPACE = "snowflake_writer"."snowflake_writer" 
+  DEFAULT_ROLE = "snowflake_writer_gcs" 
+  DEFAULT_WAREHOUSE = "snowflake_writer_gcs" 
+  DEFAULT_NAMESPACE = "snowflake_writer_gcs"."snowflake_writer_gcs" 
   MUST_CHANGE_PASSWORD = FALSE;
-GRANT ROLE "snowflake_writer" TO USER "snowflake_writer";
+GRANT ROLE "snowflake_writer_gcs" TO USER "snowflake_writer_gcs";
 ```
 
 App is developed on localhost using TDD.
 
 1. Clone from repository: `git clone [email protected]:keboola/db-writer-snowflake.git`
 2. Change directory: `cd db-writer-snowflake`
-3. Install dependencies: `docker-compose run --rm php composer install -n`
+3. Install dependencies: `docker-compose run --rm dev composer install -n`
 4. Create `.env` file:
 ```bash
-STORAGE_API_TOKEN=
 KBC_URL=
 KBC_RUNID=
 DB_HOST=

composer.json

@@ -14,6 +14,7 @@
         "ext-PDO": "*",
         "ext-json": "*",
         "ext-mbstring": "*",
+        "ext-odbc": "*",
         "keboola/db-writer-adapter": "^0.1.0",
         "keboola/db-writer-common": "^6.0",
         "keboola/db-writer-config": "^0.1.0",
@@ -24,7 +25,6 @@
     "require-dev": {
         "cweagans/composer-patches": "^1.7",
         "keboola/coding-standard": "^15.0",
-        "keboola/csv": "^1.5",
         "keboola/datadir-tests": "^5.6",
         "keboola/storage-api-client": "^14.15",
         "phpstan/phpstan": "^1.10",
@@ -40,17 +40,14 @@
         "psr-4": {
             "Keboola\\DbWriter\\Snowflake\\Tests\\": "tests/phpunit/",
             "Keboola\\DbWriter\\Snowflake\\TraitTests\\": "tests/traits/",
-            "Keboola\\DbWriter\\Snowflake\\FunctionalTests\\": "tests/functional/",
-            "Keboola\\DbWriter\\Snowflake\\PrepareTestsData\\": "tests/prepare-data/"
+            "Keboola\\DbWriter\\Snowflake\\FunctionalTests\\": "tests/functional/"
         }
     },
     "scripts": {
         "tests": [
-            "@tests-prepare-data",
             "@tests-phpunit",
             "@tests-datadir"
         ],
-        "tests-prepare-data": "php ./tests/prepare-data/prepareData.php",
         "tests-phpunit": "phpunit --testsuite unit",
         "tests-datadir": "phpunit --testsuite functional",
         "tests-perf": "phpunit --testsuite perf",