From 05fbd4305b9fb2ef71d093d08898f57726b061d0 Mon Sep 17 00:00:00 2001
From: revi cheng <revi.cheng@snowflake.com>
Date: Mon, 25 Sep 2023 12:45:21 -0700
Subject: [PATCH] GH actions tests upgrade python from 3.6 to 3.9 (#708)

---
 .github/workflows/IntegrationTest.yml | 74 +++++++++++++++++++++++++++
 .github/workflows/PerfTest.yml        | 52 +++++++++++++++++++
 2 files changed, 126 insertions(+)
 create mode 100644 .github/workflows/IntegrationTest.yml
 create mode 100644 .github/workflows/PerfTest.yml

diff --git a/.github/workflows/IntegrationTest.yml b/.github/workflows/IntegrationTest.yml
new file mode 100644
index 000000000..4be9b7545
--- /dev/null
+++ b/.github/workflows/IntegrationTest.yml
@@ -0,0 +1,74 @@
+name: Kafka Connector Integration Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: '**'
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast
+      matrix:
+        snowflake_cloud: [ 'AWS', 'AZURE', 'GCS' ]
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+    - name: "Install Java 8"
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: "Cache local Maven repository"
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        architecture: 'x64'
+    - name: Decrypt profile.json in Snowflake Cloud ${{ matrix.snowflake_cloud }}
+      run: ./.github/scripts/decrypt_secret.sh ${{ matrix.snowflake_cloud }}
+      env:
+        SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+    - name: Install Dependency
+      run: |
+        pip3 install --upgrade setuptools
+        pip3 install requests certifi "confluent-kafka[avro,json,protobuf]==1.7.0"
+        pip3 install avro-python3 kafka-python
+        pip3 install protobuf
+        pip3 install --upgrade snowflake-connector-python==2.7.4
+        curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+        sudo apt-get -y install jq vim
+        sudo apt-get -y install protobuf-compiler
+
+    - name: Install Squid as Proxy Server and Apache Utils for Password Authentication
+      run: |
+        sudo apt-get update
+        sudo apt-get -y install squid3
+        sudo apt-get install apache2-utils
+
+    - name: Change squid config and run Proxy Server
+      run: |
+        sudo touch /etc/squid/passwords
+        sudo chmod 777 /etc/squid/passwords
+        sudo htpasswd -db -c /etc/squid/passwords admin test
+        sudo mv .github/scripts/squid.conf /etc/squid/squid.conf
+        sudo service squid start
+    
+    - name: Build with Unit and Integration Test against Snowflake Cloud in ${{ matrix.snowflake_cloud }}
+      env:
+        JACOCO_COVERAGE: true
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+        SHELL: "/bin/bash"
+      run: |
+        cd test
+        ./build_runtime_jar.sh ../../snowflake-kafka-connector
+      
+    - name: Code Coverage
+      uses: codecov/codecov-action@v1
\ No newline at end of file
diff --git a/.github/workflows/PerfTest.yml b/.github/workflows/PerfTest.yml
new file mode 100644
index 000000000..a3a7b75f8
--- /dev/null
+++ b/.github/workflows/PerfTest.yml
@@ -0,0 +1,52 @@
+name: Kafka Connector Performance Test
+
+on:
+  schedule:
+    - cron:  '0 6 * * *' # at 6.00 every day
+
+jobs:
+  build:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout Code
+      uses: actions/checkout@v2
+    - name: "Install Java 8"
+      uses: actions/setup-java@v1
+      with:
+        java-version: 1.8
+    - name: "Cache local Maven repository"
+      uses: actions/cache@v2
+      with:
+        path: ~/.m2/repository
+        key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          ${{ runner.os }}-maven-
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+        architecture: 'x64'
+    - name: Decrypt snowflake.json
+      run: ./.github/scripts/perf_test_decrypt_secret.sh
+      env:
+        SNOWFLAKE_TEST_PROFILE_SECRET: ${{ secrets.SNOWFLAKE_TEST_PROFILE_SECRET }}
+    - name: Install Dependency
+      run: |
+        pip3 install --upgrade setuptools
+        pip3 install "confluent-kafka[avro]"
+        pip3 install avro-python3
+        pip3 install --upgrade snowflake-connector-python==2.7.4
+        pip3 install pycrypto
+
+    - name: Perf Test
+      env:
+        SNOWFLAKE_CREDENTIAL_FILE: "../profile.json"
+      run: |
+        cd test/perf_test
+        ./scripts/run.sh
+
+    - name: Code Coverage
+      uses: codecov/codecov-action@v1
+
+
+