GMT-109: Initial copy

.dockerignore

@@ -1 +1,7 @@
 vendor/
+/data/
+tests/data/out/
+.idea/
+.env
+/build
+/.phpunit.result.cache

.env.dist

@@ -0,0 +1,10 @@
+STORAGE_API_TOKEN=
+KBC_URL=
+KBC_RUNID=
+DB_HOST=
+DB_PORT=
+DB_DATABASE=
+DB_USER=
+DB_PASSWORD=
+DB_SCHEMA=
+DB_WAREHOUSE=

.github/workflows/push.yml

@@ -1,106 +1,95 @@
 name: 'GitHub Actions'
 'on':
-    - push
+  - push
 concurrency: 'ci-${{ github.ref }}'
 env:
-    APP_IMAGE: keboola-component
-    KBC_DEVELOPERPORTAL_VENDOR: keboola
-    KBC_DEVELOPERPORTAL_APP: keboola.db-writer-snowflake-gcs
-    KBC_DEVELOPERPORTAL_USERNAME: keboola+wr_db_snowflake_gcs
-    KBC_DEVELOPERPORTAL_PASSWORD: '${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}'
-    DOCKERHUB_USER: ''
-    DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}'
-    KBC_STORAGE_TOKEN: '${{ secrets.KBC_STORAGE_TOKEN }}'
-    KBC_TEST_PROJECT_URL: ''
-    KBC_TEST_PROJECT_CONFIGS: ''
+  APP_IMAGE: keboola-component
+  KBC_DEVELOPERPORTAL_VENDOR: keboola
+  KBC_DEVELOPERPORTAL_APP: keboola.db-writer-snowflake-gcs
+  KBC_DEVELOPERPORTAL_USERNAME: keboola+wr_db_snowflake_gcs
+  KBC_DEVELOPERPORTAL_PASSWORD: '${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }}'
+  DOCKERHUB_USER: ''
+  DOCKERHUB_TOKEN: '${{ secrets.DOCKERHUB_TOKEN }}'
+  KBC_STORAGE_TOKEN: '${{ secrets.KBC_STORAGE_TOKEN }}'
+  KBC_TEST_PROJECT_URL: ''
+  KBC_TEST_PROJECT_CONFIGS: ''
 jobs:
-    build:
-        runs-on: ubuntu-latest
-        outputs:
-            app_image_tag: '${{ steps.tag.outputs.app_image_tag }}'
-            is_semantic_tag: '${{ steps.tag.outputs.is_semantic_tag }}'
-        steps:
-            -
-                name: 'Check out the repo'
-                uses: actions/checkout@v3
-            -
-                name: 'Print Docker version'
-                run: 'docker -v'
-            -
-                name: 'Docker login'
-                if: env.DOCKERHUB_TOKEN
-                run: 'docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"'
-            -
-                name: 'Build image'
-                run: 'docker build -t $APP_IMAGE .'
-            -
-                name: 'Set image tag'
-                id: tag
-                run: |
-                    TAG="${GITHUB_REF##*/}"
-                    IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false)
-                    echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'"
-                    echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT
-                    echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT
-            -
-                name: 'Push image to ECR'
-                uses: keboola/action-push-to-ecr@master
-                with:
-                    vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
-                    app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-                    username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
-                    password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
-                    tag: '${{ steps.tag.outputs.app_image_tag }}'
-                    push_latest: '${{ steps.tag.outputs.is_semantic_tag }}'
-                    source_image: '${{ env.APP_IMAGE}}'
-    tests:
-        needs: build
-        runs-on: ubuntu-latest
-        steps:
-            -
-                name: 'Check out the repo'
-                uses: actions/checkout@v3
-            -
-                name: 'Pull image from ECR'
-                uses: keboola/action-pull-from-ecr@master
-                with:
-                    vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
-                    app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-                    username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
-                    password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
-                    tag: '${{ needs.build.outputs.app_image_tag }}'
-                    target_image: '${{ env.APP_IMAGE}}'
-                    tag_as_latest: true
-            -
-                name: 'Run tests'
-                run: 'docker run ${{env.APP_IMAGE}} composer ci'
-    tests-in-kbc:
-        needs: build
-        runs-on: ubuntu-latest
-        steps:
-            -
-                name: 'Run KBC test jobs'
-                if: 'env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS'
-                uses: keboola/action-run-configs-parallel@master
-                with:
-                    token: '${{ env.KBC_STORAGE_TOKEN }}'
-                    componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-                    tag: '${{ needs.build.outputs.app_image_tag }}'
-                    configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}'
-    deploy:
-        needs:
-            - build
-            - tests
-            - tests-in-kbc
-        runs-on: ubuntu-latest
-        if: 'startsWith(github.ref, ''refs/tags/'') && needs.build.outputs.is_semantic_tag == ''true'''
-        steps:
-            -
-                name: 'Set tag in the Developer Portal'
-                uses: keboola/action-set-tag-developer-portal@master
-                with:
-                    vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
-                    app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
-                    username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
-                    password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
-                    tag: '${{ needs.build.outputs.app_image_tag }}'
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      app_image_tag: '${{ steps.tag.outputs.app_image_tag }}'
+      is_semantic_tag: '${{ steps.tag.outputs.is_semantic_tag }}'
+    steps:
+      - name: 'Check out the repo'
+        uses: actions/checkout@v3
+      - name: 'Print Docker version'
+        run: 'docker -v'
+      - name: 'Docker login'
+        if: env.DOCKERHUB_TOKEN
+        run: 'docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"'
+      - name: 'Build image'
+        run: 'docker build -t $APP_IMAGE .'
+      - name: 'Set image tag'
+        id: tag
+        run: |
+          TAG="${GITHUB_REF##*/}"
+          IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false)
+          echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'"
+          echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT
+          echo "app_image_tag=$TAG" >> $GITHUB_OUTPUT
+      - name: 'Push image to ECR'
+        uses: keboola/action-push-to-ecr@master
+        with:
+          vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
+          app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
+          username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
+          password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
+          tag: '${{ steps.tag.outputs.app_image_tag }}'
+          push_latest: '${{ steps.tag.outputs.is_semantic_tag }}'
+          source_image: '${{ env.APP_IMAGE}}'
+  tests:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Check out the repo'
+        uses: actions/checkout@v3
+      - name: 'Pull image from ECR'
+        uses: keboola/action-pull-from-ecr@master
+        with:
+          vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
+          app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
+          username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
+          password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
+          tag: '${{ needs.build.outputs.app_image_tag }}'
+          target_image: '${{ env.APP_IMAGE}}'
+          tag_as_latest: true
+      - name: 'Run tests'
+        run: 'docker run ${{env.APP_IMAGE}} composer ci'
+  tests-in-kbc:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: 'Run KBC test jobs'
+        if: 'env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS'
+        uses: keboola/action-run-configs-parallel@master
+        with:
+          token: '${{ env.KBC_STORAGE_TOKEN }}'
+          componentId: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
+          tag: '${{ needs.build.outputs.app_image_tag }}'
+          configs: '${{ env.KBC_TEST_PROJECT_CONFIGS }}'
+  deploy:
+    needs:
+      - build
+      - tests
+      - tests-in-kbc
+    runs-on: ubuntu-latest
+    if: 'startsWith(github.ref, ''refs/tags/'') && needs.build.outputs.is_semantic_tag == ''true'''
+    steps:
+      - name: 'Set tag in the Developer Portal'
+        uses: keboola/action-set-tag-developer-portal@master
+        with:
+          vendor: '${{ env.KBC_DEVELOPERPORTAL_VENDOR }}'
+          app_id: '${{ env.KBC_DEVELOPERPORTAL_APP }}'
+          username: '${{ env.KBC_DEVELOPERPORTAL_USERNAME }}'
+          password: '${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}'
+          tag: '${{ needs.build.outputs.app_image_tag }}'

.gitignore

@@ -1,3 +1,6 @@
-/vendor
-/.idea
-/data
+vendor/
+/data/
+.idea/
+.env
+/.phpunit.result.cache
+/tests/prepare-data/manifestData.json

Dockerfile

@@ -1,40 +1,77 @@
-FROM php:8-cli
+FROM php:8.2-cli-buster
 
-ARG COMPOSER_FLAGS="--prefer-dist --no-interaction"
-ARG DEBIAN_FRONTEND=noninteractive
-ENV COMPOSER_ALLOW_SUPERUSER 1
-ENV COMPOSER_PROCESS_TIMEOUT 3600
+ARG SNOWFLAKE_ODBC_VERSION=2.25.10
+ARG SNOWFLAKE_SNOWSQL_VERSION=1.2.10
+ARG SNOWFLAKE_ODBC_GPG_KEY=630D9F3CAB551AF3
+ARG SNOWFLAKE_SNOWSQL_GPG_KEY=EC218558EABB25A1
+ENV COMPOSER_ALLOW_SUPERUSER=1
+ENV DEBIAN_FRONTEND noninteractive
+ENV TMPDIR=/opt/snowsqltempdir
 
-WORKDIR /code/
+RUN mkdir -p /opt/snowsqltempdir
 
-COPY docker/php-prod.ini /usr/local/etc/php/php.ini
-COPY docker/composer-install.sh /tmp/composer-install.sh
+# Install Dependencies
+RUN apt-get update \
+  && apt-get install unzip git unixodbc unixodbc-dev libpq-dev debsig-verify libicu-dev -y
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        git \
-        locales \
-        unzip \
-	&& rm -r /var/lib/apt/lists/* \
-	&& sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \
-	&& locale-gen \
-	&& chmod +x /tmp/composer-install.sh \
-	&& /tmp/composer-install.sh
+RUN docker-php-ext-configure intl \
+    && docker-php-ext-install intl
 
-ENV LANGUAGE=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LC_ALL=en_US.UTF-8
+# Install PHP odbc extension
+# https://github.com/docker-library/php/issues/103
+RUN set -x \
+    && docker-php-source extract \
+    && cd /usr/src/php/ext/odbc \
+    && phpize \
+    && sed -ri 's@^ *test +"\$PHP_.*" *= *"no" *&& *PHP_.*=yes *$@#&@g' configure \
+    && ./configure --with-unixODBC=shared,/usr \
+    && docker-php-ext-install odbc \
+    && docker-php-source delete
 
-## Composer - deps always cached unless changed
-# First copy only composer files
-COPY composer.* /code/
+#snoflake download + verify package
+COPY driver/snowflake-odbc-policy.pol /etc/debsig/policies/$SNOWFLAKE_ODBC_GPG_KEY/generic.pol
+COPY driver/snowflake-snowsql-policy.pol /etc/debsig/policies/$SNOWFLAKE_SNOWSQL_GPG_KEY/generic.pol
+COPY driver/simba.snowflake.ini /usr/lib/snowflake/odbc/lib/simba.snowflake.ini
+ADD https://sfc-repo.azure.snowflakecomputing.com/odbc/linux/$SNOWFLAKE_ODBC_VERSION/snowflake-odbc-$SNOWFLAKE_ODBC_VERSION.x86_64.deb /tmp/snowflake-odbc.deb
+RUN curl -o /usr/bin/snowsql-linux_x86_64.bash https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-$SNOWFLAKE_SNOWSQL_VERSION-linux_x86_64.bash
+RUN curl -o /tmp/snowsql-linux_x86_64.bash.sig https://sfc-repo.snowflakecomputing.com/snowsql/bootstrap/1.2/linux_x86_64/snowsql-$SNOWFLAKE_SNOWSQL_VERSION-linux_x86_64.bash.sig
 
-# Download dependencies, but don't run scripts or init autoloaders as the app is missing
-RUN composer install $COMPOSER_FLAGS --no-scripts --no-autoloader
+# snowflake - charset settings
+ENV LANG en_US.UTF-8
+ENV LC_ALL=C.UTF-8
 
-# Copy rest of the app
-COPY . /code/
+RUN mkdir -p ~/.gnupg \
+    && chmod 700 ~/.gnupg \
+    && echo "disable-ipv6" >> ~/.gnupg/dirmngr.conf \
+    && mkdir /usr/share/debsig/keyrings/$SNOWFLAKE_ODBC_GPG_KEY \
+    && mkdir /usr/share/debsig/keyrings/$SNOWFLAKE_SNOWSQL_GPG_KEY \
+    && if ! gpg --keyserver hkp://keys.gnupg.net --recv-keys $SNOWFLAKE_ODBC_GPG_KEY; then \
+            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $SNOWFLAKE_ODBC_GPG_KEY;  \
+        fi \
+    && if ! gpg --keyserver hkp://keys.gnupg.net --recv-keys $SNOWFLAKE_SNOWSQL_GPG_KEY; then \
+            gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys $SNOWFLAKE_SNOWSQL_GPG_KEY;  \
+        fi \
+    && gpg --export $SNOWFLAKE_ODBC_GPG_KEY > /usr/share/debsig/keyrings/$SNOWFLAKE_ODBC_GPG_KEY/debsig.gpg \
+    && gpg --export $SNOWFLAKE_SNOWSQL_GPG_KEY > /usr/share/debsig/keyrings/$SNOWFLAKE_SNOWSQL_GPG_KEY/debsig.gpg \
+    && debsig-verify /tmp/snowflake-odbc.deb \
+    && gpg --verify /tmp/snowsql-linux_x86_64.bash.sig /usr/bin/snowsql-linux_x86_64.bash \
+    && gpg --batch --delete-key --yes $SNOWFLAKE_ODBC_GPG_KEY \
+    && gpg --batch --delete-key --yes $SNOWFLAKE_SNOWSQL_GPG_KEY \
+    && dpkg -i /tmp/snowflake-odbc.deb \
+    && SNOWSQL_DEST=/usr/bin SNOWSQL_LOGIN_SHELL=~/.profile bash /usr/bin/snowsql-linux_x86_64.bash
 
-# Run normal composer - all deps are cached already
-RUN composer install $COMPOSER_FLAGS
+RUN snowsql -v $SNOWFLAKE_SNOWSQL_VERSION
 
-CMD ["php", "/code/src/run.php"]
+# install composer
+RUN cd \
+  && curl -sS https://getcomposer.org/installer | php \
+  && ln -s /root/composer.phar /usr/local/bin/composer
+
+ADD . /code
+WORKDIR /code
+
+RUN echo "memory_limit = -1" >> /usr/local/etc/php/conf.d/php.ini
+RUN echo "date.timezone = \"Europe/Prague\"" >> /usr/local/etc/php/conf.d/php.ini
+RUN composer install --no-interaction
+
+CMD php ./src/run.php