Merge branch 'master' into ddavydov/#1046-source-twilio-fix-date-ranges

airbytehq · Nov 29, 2022 · 5ce21bc · 5ce21bc
2 parents 49149ef + 494349f
commit 5ce21bc
Show file tree

Hide file tree

Showing 54 changed files with 765 additions and 1,770 deletions.
diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml
@@ -134,20 +134,35 @@ jobs:
           EOF
 
       - name: Format
-        run: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Ensure no file change
         run: ./tools/bin/check_for_file_changes
 
       - name: Build
-        run: |
-          SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run integration tests
-        run: ./tools/bin/integration_tests_octavia.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/integration_tests_octavia.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
   # Connectors Base
   # In case of self-hosted EC2 errors, remove this block.
@@ -230,13 +245,25 @@ jobs:
           EOF
 
       - name: Generate Template scaffold
-        run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Format
-        run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Build
-        run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
@@ -392,7 +419,11 @@ jobs:
           EOF
 
       - name: Build :airbyte-webapp
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Publish Storybook to Chromatic
         uses: chromaui/action@v1
@@ -447,12 +478,20 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run End-to-End Frontend Tests
         env:
           CYPRESS_WEBAPP_KEY: ${{ secrets.CYPRESS_WEBAPP_KEY }}
-        run: ./tools/bin/e2e_test.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/e2e_test.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
   # In case of self-hosted EC2 errors, remove this block.
   stop-frontend-runner:
     name: "Frontend: Stop Runner"
@@ -565,25 +604,43 @@ jobs:
           EOF
 
       - name: Format
-        run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
       - name: Build
-        run: |
-          SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Integration test
-        run: |
-          SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Slow integration test
         if: contains(github.ref, 'bump-version') || contains(github.ref, 'master')
-        run: SUB_BUILD=PLATFORM ./gradlew slowIntegrationTest
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew slowIntegrationTest
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Test if Seed spec is updated
-        run: SUB_BUILD=PLATFORM ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       # todo (cgardens) - scope by platform.
       - name: Check documentation
@@ -595,10 +652,18 @@ jobs:
       #        run: docker run --rm busybox df -h
 
       - name: Run End-to-End Acceptance Tests
-        run: ./tools/bin/acceptance_test.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/acceptance_test.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Automatic Migration Acceptance Test
-        run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - uses: actions/setup-python@v4
         if: always()
@@ -792,7 +857,11 @@ jobs:
           CHANGE_MINIKUBE_NONE_USER: true
 
       - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run Kubernetes End-to-End Acceptance Tests
         env:
@@ -802,8 +871,11 @@ jobs:
           AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
-        run: |
-          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - uses: actions/setup-python@v4
         if: always()
@@ -961,14 +1033,17 @@ jobs:
           fetch-depth: 0
 
       - name: Delete default old docker and replace it with a new one
-        shell: bash
-        run: |
-          sudo rm /var/lib/dpkg/lock
-          sudo rm /var/lib/dpkg/lock-frontend
-          sudo apt-get remove docker.io || sudo apt-get remove docker
-          curl -fsSL https://get.docker.com | bash -
-          sudo rm -f /var/lib/dpkg/lock
-          sudo rm -f /var/lib/dpkg/lock-frontend
+        uses: Wandalen/wretry.action@master
+        with:
+          command: |
+            sudo rm /var/lib/dpkg/lock
+            sudo rm /var/lib/dpkg/lock-frontend
+            sudo apt-get remove docker.io || sudo apt-get remove docker
+            curl -fsSL https://get.docker.com | bash -
+            sudo rm -f /var/lib/dpkg/lock
+            sudo rm -f /var/lib/dpkg/lock-frontend
+          attempt_limit: 3
+          attempt_delay: 2000 # in ms
 
       - name: Cache Build Artifacts
         uses: ./.github/actions/cache-build-artifacts
@@ -1004,13 +1079,16 @@ jobs:
 
       - name: Install tmate and it's dependencies
         if: inputs.debug_mode == 'true'
-        shell: bash
-        run: |
-          sudo rm /var/lib/dpkg/lock
-          sudo rm /var/lib/dpkg/lock-frontend
-          sudo apt-get -y install tmate
-          sudo rm -f /var/lib/dpkg/lock
-          sudo rm -f /var/lib/dpkg/lock-frontend
+        uses: Wandalen/wretry.action@master
+        with:
+          command: |
+            sudo rm /var/lib/dpkg/lock
+            sudo rm /var/lib/dpkg/lock-frontend
+            sudo apt-get -y install tmate
+            sudo rm -f /var/lib/dpkg/lock
+            sudo rm -f /var/lib/dpkg/lock-frontend
+          attempt_limit: 3
+          attempt_delay: 2000 # in ms
 
       - name: Start tmate session in background
         if: inputs.debug_mode == 'true'
@@ -1031,7 +1109,11 @@ jobs:
           extra-setup-args: --docker --kube-apiserver-arg service-node-port-range=6000-32767
 
       - name: Try the cluster !
-        run: kubectl get pods -A
+        uses: Wandalen/wretry.action@master
+        with:
+          command: kubectl get pods -A
+          attempt_limit: 3
+          attempt_delay: 3000 # in ms
 
       - name: Set up CI Gradle Properties
         run: |
@@ -1047,7 +1129,11 @@ jobs:
           EOF
 
       - name: Build Platform Docker Images
-        run: pwd && SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+        uses: Wandalen/wretry.action@master
+        with:
+          command: pwd && SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Run Helm End-to-End Acceptance Tests
         env:
@@ -1058,8 +1144,11 @@ jobs:
           SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
           SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
         timeout-minutes: 20
-        run: |
-          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+        uses: Wandalen/wretry.action@master
+        with:
+          command: CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
 
       - name: Generate Test Report
         uses: dorny/test-reporter@v1

diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml
@@ -301,18 +301,25 @@ jobs:
           echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV
       - name: Prepare Sentry
         if: startsWith(matrix.connector, 'connectors')
-        run: |
-          curl -sL https://sentry.io/get-cli/ | bash || echo "sentry cli already installed"
+        uses: Wandalen/wretry.action@master
+        with:
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
+          command: curl -sL https://sentry.io/get-cli/ | bash || echo "sentry cli already installed"
       - name: Publish ${{ matrix.connector }}
-        run: |
-          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
-          ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
         id: publish
         env:
           DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
           DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
+        uses: Wandalen/wretry.action@master
+        with:
+          command: |
+            echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
+            ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache   
+          attempt_limit: 3
+          attempt_delay: 5000 in # ms
       - name: Create Sentry Release
         if: startsWith(matrix.connector, 'connectors') && success()
         run: |
@@ -346,8 +353,11 @@ jobs:
           fi
       - name: Run gradle process changes
         if: github.event.inputs.auto-bump-version == 'true' && success()
-        run: |
-          ./gradlew :airbyte-config:init:processResources
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./gradlew :airbyte-config:init:processResources
+          attempt_limit: 3
+          attempt_delay: 5000 # in ms
       - name: git config
         if: github.event.inputs.auto-bump-version == 'true' && success()
         run: |

diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml
@@ -123,13 +123,16 @@ jobs:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
 
       - name: Test ${{ github.event.inputs.connector }}
-        run: |
-          ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
         id: test
         env:
           ACTION_RUN_ID: ${{github.run_id}}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
+        uses: Wandalen/wretry.action@master
+        with:
+          command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
+          attempt_limit: 3
+          attempt_delay: 10000 # in ms
       - name: Archive test reports artifacts
         if: github.event.inputs.comment-id && failure()
         uses: actions/upload-artifact@v3

diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.11.0
+Declare a new authenticator `SingleUseRefreshTokenOauth2Authenticator` that can perform connector configuration mutation and emit `AirbyteControlMessage.ConnectorConfig`.
+
 ## 0.10.0
 Low-code: Add `start_from_page` option to a PageIncrement class