Merge remote-tracking branch 'origin/master' into aleonets/3931-3932-…

…fix-mysql # Conflicts: # docs/integrations/sources/mysql.md
airbytehq · Sep 23, 2021 · ea4e2cf · ea4e2cf
2 parents 35c5781 + aac98b9
commit ea4e2cf
Show file tree

Hide file tree

Showing 550 changed files with 15,240 additions and 2,532 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.29.17-alpha
+current_version = 0.29.21-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

diff --git a/.env b/.env
@@ -1,4 +1,4 @@
-VERSION=0.29.17-alpha
+VERSION=0.29.21-alpha
 
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker

diff --git a/.github/ISSUE_TEMPLATE/new-integration-request.md b/.github/ISSUE_TEMPLATE/new-integration-request.md
@@ -12,9 +12,10 @@ assignees: ''
 * Do you need a specific version of the underlying data source e.g: you specifically need support for an older version of the API or DB? 
 
 ## Describe the context around this new connector
-* Which team in your company wants this integration, what for? This helps us understand the use case.
+* Why do you need this integration? How does your team intend to use the data? This helps us understand the use case.
 * How often do you want to run syncs?
 * If this is an API source connector, which entities/endpoints do you need supported?
+* If the connector is for a paid service, can we name you as a mutual user when we subscribe for an account? Which company should we name? 
 
 ## Describe the alternative you are considering or using
 What are you considering doing if you don’t have this integration through Airbyte?

diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -36,3 +36,9 @@ CDK:
 normalization:
   - airbyte-integrations/bases/base-normalization/*
   - airbyte-integrations/bases/base-normalization/**/*
+
+kubernetes:
+  - kube/*
+  - kube/**/*
+  - charts/*
+  - charts/**/*
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -415,6 +415,7 @@ jobs:
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
     needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready
     runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+    environment: more-secrets
     name: Acceptance Tests (Kube)
     timeout-minutes: 90
     steps:
@@ -483,11 +484,25 @@ jobs:
           df -h
           docker system df
 
+      - name: Run GCP End-to-End Acceptance Tests
+        env:
+          USER: root
+          HOME: /home/runner
+          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+          SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }}
+        run: |
+          CI=true ./tools/bin/gcp_acceptance_tests.sh
+
       - name: Run Kubernetes End-to-End Acceptance Tests
         env:
           USER: root
           HOME: /home/runner
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+          SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }}
         run: |
           CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
   # In case of self-hosted EC2 errors, remove this block.

diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml
@@ -0,0 +1,74 @@
+name: Helm
+on:
+  push:
+    paths:
+      - ".github/workflows/helm.yaml"
+      - "charts/**"
+  pull_request:
+    paths:
+      - ".github/workflows/helm.yaml"
+      - "charts/**"
+jobs:
+  lint:
+    name: Lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Kubectl
+        uses: azure/setup-kubectl@v1
+      - name: Setup Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: "3.6.3"
+      - name: Lint Chart
+        working-directory: ./charts/airbyte
+        run: ./ci.sh lint
+
+  generate-docs:
+    name: Generate Docs Parameters
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - name: Checkout bitnami-labs/readme-generator-for-helm
+        uses: actions/checkout@v2
+        with:
+          repository: "bitnami-labs/readme-generator-for-helm"
+          ref: "55cab5dd2191c4ffa7245cfefa428d4d9bb12730"
+          path: readme-generator-for-helm
+      - name: Install readme-generator-for-helm dependencies
+        working-directory: readme-generator-for-helm
+        run: npm install -g
+      - name: Test can update README with generated parameters
+        working-directory: charts/airbyte
+        run: ./ci.sh check-docs-updated
+
+  install:
+    name: Install
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Kubectl
+        uses: azure/setup-kubectl@v1
+      - name: Setup Helm
+        uses: azure/setup-helm@v1
+        with:
+          version: "3.6.3"
+      - name: Setup Kind Cluster
+        uses: helm/[email protected]
+        with:
+          version: "v0.11.1"
+          image: "kindest/node:v1.21.1"
+      - name: Install airbyte chart
+        working-directory: ./charts/airbyte
+        run: ./ci.sh install
+      - if: always()
+        name: Print diagnostics
+        working-directory: ./charts/airbyte
+        run: ./ci.sh diagnostics
+      - if: success()
+        name: Test airbyte chart
+        working-directory: ./charts/airbyte
+        run: ./ci.sh test
diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -117,13 +117,15 @@ jobs:
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
           GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }}
           GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }}
+          GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }}
           HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }}
           HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
           INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }}
           INTERCOM_INTEGRATION_TEST_CREDS: ${{ secrets.INTERCOM_INTEGRATION_TEST_CREDS }}
           ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }}
           JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }}
           KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }}
+          LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }}
           LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }}
           MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }}
           MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }}
@@ -132,6 +134,8 @@ jobs:
           PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }}
           POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }}
           POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }}
+          MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }}
+          MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }}
           POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }}
           PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }}
           RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }}
@@ -179,6 +183,8 @@ jobs:
           SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }}
           SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }}
           DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }}
+          MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }}
+          MSSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MSSQL_SSH_PWD_TEST_CREDS }}
       - run: |
           echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
           ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -112,6 +112,7 @@ jobs:
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
           GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }}
           GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }}
+          GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }}
           HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }}
           HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
           INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }}
@@ -120,6 +121,7 @@ jobs:
           JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }}
           KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }}
           SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }}
+          LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }}
           LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }}
           MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }}
           MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }}
@@ -128,6 +130,8 @@ jobs:
           PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }}
           POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }}
           POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }}
+          MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }}
+          MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }}
           POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }}
           PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }}
           RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }}
@@ -174,6 +178,8 @@ jobs:
           SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }}
           SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }}
           DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }}
+          MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }}
+          MSSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MSSQL_SSH_PWD_TEST_CREDS }}
       - run: |
           ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
         name: test ${{ github.event.inputs.connector }}

diff --git a/.gitignore b/.gitignore
@@ -29,6 +29,21 @@ __pycache__
 .ipynb_checkpoints
 .pytest_
 
+# Python unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
 # dbt
 profiles.yml