From a3100ef1beb618305b4957118e2c51a8f6350940 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:50:39 +0100 Subject: [PATCH 01/84] start --- .../connectors/destination-aws-datalake/Dockerfile | 2 +- docs/integrations/destinations/aws-datalake.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile index ebe2844b7244..a036661f7da7 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile +++ b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-aws-datalake diff --git a/docs/integrations/destinations/aws-datalake.md b/docs/integrations/destinations/aws-datalake.md index 0fa59745a562..a788b8e2b95c 100644 --- a/docs/integrations/destinations/aws-datalake.md +++ b/docs/integrations/destinations/aws-datalake.md @@ -99,5 +99,6 @@ following information to configure the destination: ## Changelog +| 0.1.2 | 2022-06-16 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | | 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | \ No newline at end of file From 7a3958f7d18f9710610a1c7a19c4452e4171a896 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:54:25 +0100 Subject: [PATCH 02/84] revert --- .../connectors/destination-aws-datalake/Dockerfile | 2 +- docs/integrations/destinations/aws-datalake.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile index a036661f7da7..ebe2844b7244 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile +++ b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-aws-datalake diff --git a/docs/integrations/destinations/aws-datalake.md b/docs/integrations/destinations/aws-datalake.md index a788b8e2b95c..0fa59745a562 100644 --- a/docs/integrations/destinations/aws-datalake.md +++ b/docs/integrations/destinations/aws-datalake.md @@ -99,6 +99,5 @@ following information to configure the destination: ## Changelog -| 0.1.2 | 2022-06-16 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | | 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | \ No newline at end of file From 4575f36b0baeb665d0a236ba3bc726ad4003b32b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:55:52 +0100 Subject: [PATCH 03/84] azblob --- .../connectors/destination-azure-blob-storage/Dockerfile | 2 +- docs/integrations/destinations/azureblobstorage.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile b/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile index f0c88d716a67..95c65d89053d 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-azure-blob-storage COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-azure-blob-storage diff --git a/docs/integrations/destinations/azureblobstorage.md b/docs/integrations/destinations/azureblobstorage.md index 45e083acb083..3a6753bd9856 100644 --- a/docs/integrations/destinations/azureblobstorage.md +++ b/docs/integrations/destinations/azureblobstorage.md @@ -137,6 +137,7 @@ They will be like this in the output file: | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.4 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.2 | 2022-01-20 | [\#9682](https://github.com/airbytehq/airbyte/pull/9682) | Each data synchronization for each stream is written to a new blob to the folder with stream name. | From 9028a585f35ef32acbaf4a5fb0f25fda0cb96b88 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:57:08 +0100 Subject: [PATCH 04/84] bq --- airbyte-integrations/connectors/destination-bigquery/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index db02ad580dba..9d02050874a2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.1.8 +LABEL io.airbyte.version=1.1.9 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index b655d3a6b439..401232de4079 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -133,6 +133,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------| +| 1.1.9 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | | 1.1.7 | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | | 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | From 892037b565eb59de9c47354c489db58b437c259f Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 16:58:32 +0100 Subject: [PATCH 05/84] bq denorm --- .../connectors/destination-bigquery-denormalized/Dockerfile | 2 +- docs/integrations/destinations/bigquery.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile index ec8026955d27..832a44505bff 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=1.1.8 +LABEL io.airbyte.version=1.1.9 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index 401232de4079..0ce737895f55 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -172,6 +172,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| 1.1.9 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 1.1.8 | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | | 1.1.7 | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | | 1.1.6 | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | From 1ef7488b7cc5bb6666195ea2f39a0f513487bd56 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:00:43 +0100 Subject: [PATCH 06/84] megapublish baaaabyyyy --- .github/workflows/publish-command.yml | 34 ++++++++++++++++++++------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 60bf4453586a..6109fc8e93a5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -65,9 +65,27 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + preprocess-matrix: + needs: start-publish-image-runner + runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + outputs: + connectorjson: ${{ steps.preprocess.outputs.connectorjson }} + steps: + - id: preprocess + run: | + start="[\"" + replace="\",\"" + end="\"]" + stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')" + middle=${stripped_connector//,/$replace} + full="$start$middle$end" + echo "::set-output name=connectorjson::$full" publish-image: timeout-minutes: 240 - needs: start-publish-image-runner + needs: preprocess-matrix + strategy: + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} runs-on: ${{ needs.start-publish-image-runner.outputs.label }} environment: more-secrets steps: @@ -89,7 +107,7 @@ jobs: with: regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} + search_string: ${{ matrix.connector }} - name: Validate input workflow format if: steps.regex.outputs.first_match != github.event.inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 @@ -126,18 +144,18 @@ jobs: source venv/bin/activate tox -r -c ./tools/tox_ci.ini pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + - name: Write Integration Test Credentials for ${{ matrix.connector }} run: | source venv/bin/activate - ci_credentials ${{ github.event.inputs.connector }} + ci_credentials ${{ matrix.connector }} env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars if: startsWith(github.event.inputs.connector, 'connectors') run: | source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - name: Prepare Sentry if: startsWith(github.event.inputs.connector, 'connectors') @@ -151,10 +169,10 @@ jobs: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ github.event.inputs.connector }} + - name: Publish ${{ matrix.connector }} run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache id: publish env: DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} From c3cf966ac19fb1df9167a4a14ce68122c82e3931 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:15:18 +0100 Subject: [PATCH 07/84] fix needs --- .github/workflows/publish-command.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 6109fc8e93a5..780ca315fc11 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -82,7 +82,9 @@ jobs: echo "::set-output name=connectorjson::$full" publish-image: timeout-minutes: 240 - needs: preprocess-matrix + needs: + - start-publish-image-runner + - preprocess-matrix strategy: matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} From 2c05ac5008d6622d657d3b89584e9aea72d91e0b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:23:22 +0100 Subject: [PATCH 08/84] matrix connectors --- .github/workflows/publish-command.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 780ca315fc11..05e7aa30e99e 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -97,7 +97,7 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -111,7 +111,7 @@ jobs: regex_flags: "i" # required to be set for this plugin search_string: ${{ matrix.connector }} - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector + if: steps.regex.outputs.first_match != matrix.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v2 @@ -153,18 +153,18 @@ jobs: env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | source tools/lib/lib.sh DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - name: Prepare Sentry - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | curl -sL https://sentry.io/get-cli/ | bash - name: Create Sentry Release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing env: @@ -182,7 +182,7 @@ jobs: # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. TZ: UTC - name: Finalize Sentry release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" env: @@ -195,14 +195,14 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :rocket: Successfully published ${{github.event.inputs.connector}} + > :rocket: Successfully published ${{matrix.connector}} - name: Add Published Failure Comment if: github.event.inputs.comment-id && !success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Failed to publish ${{github.event.inputs.connector}} + > :x: Failed to publish ${{matrix.connector}} - name: Check if connector in definitions yaml if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -246,21 +246,21 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :rocket: Auto-bumped version for ${{github.event.inputs.connector}} + > :rocket: Auto-bumped version for ${{matrix.connector}} - name: Add Version Bump Failure Comment if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Couldn't auto-bump version for ${{github.event.inputs.connector}} + > :x: Couldn't auto-bump version for ${{matrix.connector}} - name: Add Final Success Comment if: github.event.inputs.comment-id && success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner: name: Stop Build EC2 Runner From 981cb6ac2a25605a4340ec07304eef631dd8cc37 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 16 Jun 2022 22:39:10 +0000 Subject: [PATCH 09/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 19231af7cb1a..ea6afc10a8b6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -1,7 +1,7 @@ - name: Azure Blob Storage destinationDefinitionId: b4c5d105-31fd-4817-96b6-cb923bfc04cb dockerRepository: airbyte/destination-azure-blob-storage - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/azureblobstorage icon: azureblobstorage.svg resourceRequirements: diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index ec97b237f846..81d82333f41e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1,7 +1,7 @@ # This file is generated by io.airbyte.config.specs.SeedConnectorSpecGenerator. # Do NOT edit this file directly. See generator class for more details. --- -- dockerImage: "airbyte/destination-azure-blob-storage:0.1.4" +- dockerImage: "airbyte/destination-azure-blob-storage:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/azureblobstorage" connectionSpecification: From 5181fa050941881ed1329e56891ff63b15dd4884 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Thu, 16 Jun 2022 23:47:19 +0100 Subject: [PATCH 10/84] dont failfast and max parallel 5 --- .github/workflows/publish-command.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 05e7aa30e99e..d882486cec3f 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -86,6 +86,8 @@ jobs: - start-publish-image-runner - preprocess-matrix strategy: + max-parallel: 5 + fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} runs-on: ${{ needs.start-publish-image-runner.outputs.label }} From 4e3575bd989553d980987f006e39f62ef6c6d10b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:12:54 +0100 Subject: [PATCH 11/84] multi runno --- .github/workflows/publish-command.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index d882486cec3f..2d9306fbb834 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -90,7 +90,9 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + runs-on: + - ${{ needs.start-publish-image-runner.outputs.label }} + - ubuntu-latest environment: more-secrets steps: - name: Link comment to workflow run From ba45af5f1237cc3824906bcb91ff323242341316 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:40:16 +0100 Subject: [PATCH 12/84] minor --- .github/workflows/publish-command.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 2d9306fbb834..e38652863358 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -24,6 +24,10 @@ on: description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" required: true default: "true" + parallel: + description: "Switching this to true will spin up 5 build agents instead of 1 and allow multi connector publishes to run in parallel" + required: true + default: "false" jobs: find_valid_pat: @@ -90,9 +94,7 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: - - ${{ needs.start-publish-image-runner.outputs.label }} - - ubuntu-latest + runs-on: ${{ needs.start-publish-image-runner.outputs.label }} environment: more-secrets steps: - name: Link comment to workflow run From b9a993944d3b59130614888d47098832385367e0 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:46:26 +0100 Subject: [PATCH 13/84] testing matrix agents --- .github/workflows/TEMP-testing-command.yml | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/workflows/TEMP-testing-command.yml diff --git a/.github/workflows/TEMP-testing-command.yml b/.github/workflows/TEMP-testing-command.yml new file mode 100644 index 000000000000..b0440673044b --- /dev/null +++ b/.github/workflows/TEMP-testing-command.yml @@ -0,0 +1,114 @@ +name: Publish Connector Image +on: + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + required: false + default: master + connector: + description: "Airbyte Connector" + required: true + run-tests: + description: "Should run tests when publishing" + required: true + default: "true" + comment-id: + description: "The comment-id of the slash command. Used to update the comment with the status." + required: false + auto-bump-version: + description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" + required: true + default: "true" + +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + ## Gradle Build + # In case of self-hosted EC2 errors, remove this block. + start-publish-image-runner: + name: Start Build EC2 Runner + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWCRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + ec2-instance-id: ${{ matrix.ec2-instance-id }} + + stop-publish-image-runner: + name: Stop Build EC2 Runner + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] + needs: + - start-publish-image-runner # required to get output from the start-runner job + # - publish-image # required to wait when the main job is done + - find_valid_pat + runs-on: ubuntu-latest + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-publish-image-runner.outputs.label }} + ec2-instance-id: ${{ matrix.ec2-instance-id }} From 93a769547badcb63bf8d7a038082b13c869e0431 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:47:24 +0100 Subject: [PATCH 14/84] name --- .github/workflows/TEMP-testing-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/TEMP-testing-command.yml b/.github/workflows/TEMP-testing-command.yml index b0440673044b..d931ef501ee8 100644 --- a/.github/workflows/TEMP-testing-command.yml +++ b/.github/workflows/TEMP-testing-command.yml @@ -1,4 +1,4 @@ -name: Publish Connector Image +name: TEMP testing command on: workflow_dispatch: inputs: From 4169452c08a1c2304717893931ec35cda9fb4165 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:51:40 +0100 Subject: [PATCH 15/84] testing multi agents --- .github/workflows/publish-command.yml | 394 ++++++++++++++------------ 1 file changed, 209 insertions(+), 185 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index e38652863358..fba4c7c1d174 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -51,11 +51,22 @@ jobs: # In case of self-hosted EC2 errors, remove this block. start-publish-image-runner: name: Start Build EC2 Runner + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] runs-on: ubuntu-latest needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -69,6 +80,8 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + ec2-instance-id: ${{ matrix.ec2-instance-id }} preprocess-matrix: needs: start-publish-image-runner runs-on: ${{ needs.start-publish-image-runner.outputs.label }} @@ -84,189 +97,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - publish-image: - timeout-minutes: 240 - needs: - - start-publish-image-runner - - preprocess-matrix - strategy: - max-parallel: 5 - fail-fast: false - matrix: - connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} - environment: more-secrets - steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true - - name: Search for valid connector name format - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - regex_flags: "i" # required to be set for this plugin - search_string: ${{ matrix.connector }} - - name: Validate input workflow format - if: steps.regex.outputs.first_match != matrix.connector - run: echo "The connector provided has an invalid format!" && exit 1 - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - token: ${{ secrets.OCTAVIA_PAT }} - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: "17" - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - name: Install Pyenv and Tox - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - pip install --quiet tox==3.24.4 - - name: Install yq - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - sudo add-apt-repository ppa:rmescandon/yq - sudo apt update - sudo apt install yq -y - - name: Test and install CI scripts - # all CI python packages have the prefix "ci_" - run: | - source venv/bin/activate - tox -r -c ./tools/tox_ci.ini - pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ matrix.connector }} - run: | - source venv/bin/activate - ci_credentials ${{ matrix.connector }} - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Set Name and Version Environment Vars - if: startsWith(matrix.connector, 'connectors') - run: | - source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - - name: Prepare Sentry - if: startsWith(matrix.connector, 'connectors') - run: | - curl -sL https://sentry.io/get-cli/ | bash - - name: Create Sentry Release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ matrix.connector }} - run: | - echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - - name: Finalize Sentry release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{matrix.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{matrix.connector}} - - name: Check if connector in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - then exit 1 - fi - - name: Bump version in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - if [ -z "$sourcename" ] - then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - fi - - name: Run gradle process changes - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - ./gradlew :airbyte-config:init:processResources - - name: git config - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git config user.name 'Octavia Squidington III' - git config user.email 'octavia-squidington-iii@users.noreply.github.com' - - name: git commit and push - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git add -u - git commit -m "auto-bump connector version" - git pull origin ${{ github.event.inputs.gitref }} - git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{matrix.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Couldn't auto-bump version for ${{matrix.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # publish-image: + # timeout-minutes: 240 + # needs: + # - start-publish-image-runner + # - preprocess-matrix + # strategy: + # max-parallel: 5 + # fail-fast: false + # matrix: + # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + # runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + # environment: more-secrets + # steps: + # - name: Link comment to workflow run + # if: github.event.inputs.comment-id + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # - name: Set up Cloud SDK + # uses: google-github-actions/setup-gcloud@v0 + # with: + # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + # export_default_credentials: true + # - name: Search for valid connector name format + # id: regex + # uses: AsasInnab/regex-action@v1 + # with: + # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + # regex_flags: "i" # required to be set for this plugin + # search_string: ${{ matrix.connector }} + # - name: Validate input workflow format + # if: steps.regex.outputs.first_match != matrix.connector + # run: echo "The connector provided has an invalid format!" && exit 1 + # - name: Checkout Airbyte + # uses: actions/checkout@v2 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # token: ${{ secrets.OCTAVIA_PAT }} + # - name: Install Java + # uses: actions/setup-java@v1 + # with: + # java-version: "17" + # - name: Install Python + # uses: actions/setup-python@v2 + # with: + # python-version: "3.9" + # - name: Install Pyenv and Tox + # run: | + # python3 -m pip install --quiet virtualenv==16.7.9 --user + # python3 -m virtualenv venv + # source venv/bin/activate + # pip install --quiet tox==3.24.4 + # - name: Install yq + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + # sudo add-apt-repository ppa:rmescandon/yq + # sudo apt update + # sudo apt install yq -y + # - name: Test and install CI scripts + # # all CI python packages have the prefix "ci_" + # run: | + # source venv/bin/activate + # tox -r -c ./tools/tox_ci.ini + # pip install --quiet -e ./tools/ci_* + # - name: Write Integration Test Credentials for ${{ matrix.connector }} + # run: | + # source venv/bin/activate + # ci_credentials ${{ matrix.connector }} + # env: + # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + # - name: Set Name and Version Environment Vars + # if: startsWith(matrix.connector, 'connectors') + # run: | + # source tools/lib/lib.sh + # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + # - name: Prepare Sentry + # if: startsWith(matrix.connector, 'connectors') + # run: | + # curl -sL https://sentry.io/get-cli/ | bash + # - name: Create Sentry Release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Publish ${{ matrix.connector }} + # run: | + # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + # id: publish + # env: + # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + # TZ: UTC + # - name: Finalize Sentry release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Add Published Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Successfully published ${{matrix.connector}} + # - name: Add Published Failure Comment + # if: github.event.inputs.comment-id && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Failed to publish ${{matrix.connector}} + # - name: Check if connector in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + # then exit 1 + # fi + # - name: Bump version in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + # if [ -z "$sourcename" ] + # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + # fi + # - name: Run gradle process changes + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # ./gradlew :airbyte-config:init:processResources + # - name: git config + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git config user.name 'Octavia Squidington III' + # git config user.email 'octavia-squidington-iii@users.noreply.github.com' + # - name: git commit and push + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git add -u + # git commit -m "auto-bump connector version" + # git pull origin ${{ github.event.inputs.gitref }} + # git push origin ${{ github.event.inputs.gitref }} + # - name: Add Version Bump Success Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Auto-bumped version for ${{matrix.connector}} + # - name: Add Version Bump Failure Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Couldn't auto-bump version for ${{matrix.connector}} + # - name: Add Final Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner: name: Stop Build EC2 Runner @@ -274,6 +287,17 @@ jobs: - start-publish-image-runner # required to get output from the start-runner job - publish-image # required to wait when the main job is done - find_valid_pat + strategy: + fail-fast: false + matrix: + ec2-instance-id: + [ + "${{ github.run_id }}-${{ github.run_attempt }}-1", + "${{ github.run_id }}-${{ github.run_attempt }}-2", + "${{ github.run_id }}-${{ github.run_attempt }}-3", + "${{ github.run_id }}-${{ github.run_attempt }}-4", + "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + ] runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: @@ -289,4 +313,4 @@ jobs: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} label: ${{ needs.start-publish-image-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }} + ec2-instance-id: ${{ matrix.ec2-instance-id }} From ec764903a5bf19f8eaab03f43c4fba094d8e03d2 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 10:53:56 +0100 Subject: [PATCH 16/84] tmp fix --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index fba4c7c1d174..3a80e5618089 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -285,7 +285,7 @@ jobs: name: Stop Build EC2 Runner needs: - start-publish-image-runner # required to get output from the start-runner job - - publish-image # required to wait when the main job is done + # - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false From 2da540a3e7945a58095a244110732dc75f4e77a7 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:11:56 +0100 Subject: [PATCH 17/84] new multi agents --- .github/workflows/publish-command.yml | 116 +++++++++++++++++++++----- 1 file changed, 96 insertions(+), 20 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 3a80e5618089..8877e27222cc 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -49,24 +49,101 @@ jobs: ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner: - name: Start Build EC2 Runner - strategy: - fail-fast: false - matrix: - ec2-instance-id: - [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", - ] + start-publish-image-runner-0: + name: Start Build EC2 Runner 0 runs-on: ubuntu-latest needs: find_valid_pat outputs: label: ${{ steps.start-ec2-runner.outputs.label }} - # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-1: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 1 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-2: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 2 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-3: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 3 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: airbyte-publisher-runner + start-publish-image-runner-4: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 4 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - name: Checkout Airbyte uses: actions/checkout@v2 @@ -81,7 +158,6 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} label: airbyte-publisher-runner - ec2-instance-id: ${{ matrix.ec2-instance-id }} preprocess-matrix: needs: start-publish-image-runner runs-on: ${{ needs.start-publish-image-runner.outputs.label }} @@ -292,11 +368,11 @@ jobs: matrix: ec2-instance-id: [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", + "${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", + "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", ] runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs From 5f0c38b53db089d79bcf3322b27bf96c6e986bfe Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:18:09 +0100 Subject: [PATCH 18/84] multi test --- .github/workflows/publish-command.yml | 37 +++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 8877e27222cc..98804ef50bdf 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -357,10 +357,39 @@ jobs: # body: | # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. - stop-publish-image-runner: + stop-publish-image-runner-0: + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs name: Stop Build EC2 Runner needs: - - start-publish-image-runner # required to get output from the start-runner job + - start-publish-image-runner-0 # required to get output from the start-runner job + - preprocess-matrix + # - publish-image # required to wait when the main job is done + - find_valid_pat + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: supertopher/ec2-github-runner@base64v1.0.10 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-publish-image-runner-0.outputs.label }} + ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} + stop-publish-image-runner-multi: + if: github.event.inputs.parallel == 'true' + name: Stop Build EC2 Runner + needs: + - start-publish-image-runner-0 + - start-publish-image-runner-1 + - start-publish-image-runner-2 + - start-publish-image-runner-3 + - start-publish-image-runner-4 + - preprocess-matrix # - publish-image # required to wait when the main job is done - find_valid_pat strategy: @@ -368,14 +397,12 @@ jobs: matrix: ec2-instance-id: [ - "${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", ] runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -388,5 +415,5 @@ jobs: with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner.outputs.label }} + label: ${{ needs.start-publish-image-runner-0.outputs.label }} ec2-instance-id: ${{ matrix.ec2-instance-id }} From 3849459f393111bf98dbf96e5d76351a912f806f Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:21:39 +0100 Subject: [PATCH 19/84] tryy --- .github/workflows/publish-command.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 98804ef50bdf..454991227c39 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -159,8 +159,8 @@ jobs: github-token: ${{ needs.find_valid_pat.outputs.pat }} label: airbyte-publisher-runner preprocess-matrix: - needs: start-publish-image-runner - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} outputs: connectorjson: ${{ steps.preprocess.outputs.connectorjson }} steps: @@ -176,14 +176,14 @@ jobs: # publish-image: # timeout-minutes: 240 # needs: - # - start-publish-image-runner + # - start-publish-image-runner-0 # - preprocess-matrix # strategy: # max-parallel: 5 # fail-fast: false # matrix: # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: ${{ needs.start-publish-image-runner.outputs.label }} + # runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} # environment: more-secrets # steps: # - name: Link comment to workflow run From 2a58e3d7be95b5122f81fc9963d9d5e825c9949b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 11:30:25 +0100 Subject: [PATCH 20/84] let's do this --- .github/workflows/publish-command.yml | 370 +++++++++++++------------- 1 file changed, 185 insertions(+), 185 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 454991227c39..b59121070189 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -173,189 +173,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - # publish-image: - # timeout-minutes: 240 - # needs: - # - start-publish-image-runner-0 - # - preprocess-matrix - # strategy: - # max-parallel: 5 - # fail-fast: false - # matrix: - # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} - # environment: more-secrets - # steps: - # - name: Link comment to workflow run - # if: github.event.inputs.comment-id - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - # - name: Set up Cloud SDK - # uses: google-github-actions/setup-gcloud@v0 - # with: - # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - # export_default_credentials: true - # - name: Search for valid connector name format - # id: regex - # uses: AsasInnab/regex-action@v1 - # with: - # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - # regex_flags: "i" # required to be set for this plugin - # search_string: ${{ matrix.connector }} - # - name: Validate input workflow format - # if: steps.regex.outputs.first_match != matrix.connector - # run: echo "The connector provided has an invalid format!" && exit 1 - # - name: Checkout Airbyte - # uses: actions/checkout@v2 - # with: - # repository: ${{ github.event.inputs.repo }} - # ref: ${{ github.event.inputs.gitref }} - # token: ${{ secrets.OCTAVIA_PAT }} - # - name: Install Java - # uses: actions/setup-java@v1 - # with: - # java-version: "17" - # - name: Install Python - # uses: actions/setup-python@v2 - # with: - # python-version: "3.9" - # - name: Install Pyenv and Tox - # run: | - # python3 -m pip install --quiet virtualenv==16.7.9 --user - # python3 -m virtualenv venv - # source venv/bin/activate - # pip install --quiet tox==3.24.4 - # - name: Install yq - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - # sudo add-apt-repository ppa:rmescandon/yq - # sudo apt update - # sudo apt install yq -y - # - name: Test and install CI scripts - # # all CI python packages have the prefix "ci_" - # run: | - # source venv/bin/activate - # tox -r -c ./tools/tox_ci.ini - # pip install --quiet -e ./tools/ci_* - # - name: Write Integration Test Credentials for ${{ matrix.connector }} - # run: | - # source venv/bin/activate - # ci_credentials ${{ matrix.connector }} - # env: - # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - # - name: Set Name and Version Environment Vars - # if: startsWith(matrix.connector, 'connectors') - # run: | - # source tools/lib/lib.sh - # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - # - name: Prepare Sentry - # if: startsWith(matrix.connector, 'connectors') - # run: | - # curl -sL https://sentry.io/get-cli/ | bash - # - name: Create Sentry Release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Publish ${{ matrix.connector }} - # run: | - # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - # id: publish - # env: - # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - # TZ: UTC - # - name: Finalize Sentry release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Add Published Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Successfully published ${{matrix.connector}} - # - name: Add Published Failure Comment - # if: github.event.inputs.comment-id && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Failed to publish ${{matrix.connector}} - # - name: Check if connector in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - # then exit 1 - # fi - # - name: Bump version in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - # if [ -z "$sourcename" ] - # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - # fi - # - name: Run gradle process changes - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # ./gradlew :airbyte-config:init:processResources - # - name: git config - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git config user.name 'Octavia Squidington III' - # git config user.email 'octavia-squidington-iii@users.noreply.github.com' - # - name: git commit and push - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git add -u - # git commit -m "auto-bump connector version" - # git pull origin ${{ github.event.inputs.gitref }} - # git push origin ${{ github.event.inputs.gitref }} - # - name: Add Version Bump Success Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Auto-bumped version for ${{matrix.connector}} - # - name: Add Version Bump Failure Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Couldn't auto-bump version for ${{matrix.connector}} - # - name: Add Final Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + publish-image: + timeout-minutes: 240 + needs: + - start-publish-image-runner-0 + - preprocess-matrix + strategy: + max-parallel: 5 + fail-fast: false + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + environment: more-secrets + steps: + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ matrix.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != matrix.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.OCTAVIA_PAT }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: "17" + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Pyenv and Tox + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Install yq + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt update + sudo apt install yq -y + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ matrix.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ matrix.connector }} + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + - name: Set Name and Version Environment Vars + if: startsWith(matrix.connector, 'connectors') + run: | + source tools/lib/lib.sh + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + - name: Prepare Sentry + if: startsWith(matrix.connector, 'connectors') + run: | + curl -sL https://sentry.io/get-cli/ | bash + - name: Create Sentry Release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Publish ${{ matrix.connector }} + run: | + echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + id: publish + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + - name: Finalize Sentry release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Add Published Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Successfully published ${{matrix.connector}} + - name: Add Published Failure Comment + if: github.event.inputs.comment-id && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Failed to publish ${{matrix.connector}} + - name: Check if connector in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + then exit 1 + fi + - name: Bump version in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + if [ -z "$sourcename" ] + then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + fi + - name: Run gradle process changes + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + ./gradlew :airbyte-config:init:processResources + - name: git config + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git config user.name 'Octavia Squidington III' + git config user.email 'octavia-squidington-iii@users.noreply.github.com' + - name: git commit and push + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git add -u + git commit -m "auto-bump connector version" + git pull origin ${{ github.event.inputs.gitref }} + git push origin ${{ github.event.inputs.gitref }} + - name: Add Version Bump Success Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Auto-bumped version for ${{matrix.connector}} + - name: Add Version Bump Failure Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Couldn't auto-bump version for ${{matrix.connector}} + - name: Add Final Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -363,7 +363,7 @@ jobs: needs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat runs-on: ubuntu-latest steps: @@ -390,7 +390,7 @@ jobs: - start-publish-image-runner-3 - start-publish-image-runner-4 - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false From db58ff42e69d06696e1c2193102b46258fe8c89f Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:08:06 +0100 Subject: [PATCH 21/84] magico --- .github/workflows/publish-command.yml | 30 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index b59121070189..1c74f63d4373 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -183,7 +183,7 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + runs-on: self-hosted environment: more-secrets steps: - name: Link comment to workflow run @@ -381,7 +381,7 @@ jobs: label: ${{ needs.start-publish-image-runner-0.outputs.label }} ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} stop-publish-image-runner-multi: - if: github.event.inputs.parallel == 'true' + if: ${{ always() && github.event.inputs.parallel == 'true' }} name: Stop Build EC2 Runner needs: - start-publish-image-runner-0 @@ -395,12 +395,24 @@ jobs: strategy: fail-fast: false matrix: - ec2-instance-id: + ec2-instance: [ - "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", - "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", - "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", - "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, ] runs-on: ubuntu-latest steps: @@ -415,5 +427,5 @@ jobs: with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner-0.outputs.label }} - ec2-instance-id: ${{ matrix.ec2-instance-id }} + label: ${{ matrix.ec2-instance.label }} + ec2-instance-id: ${{ matrix.ec2-instance.id }} From d1cf7f20fd45b4a64c4b893c04902f0204de50f5 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:28:26 +0100 Subject: [PATCH 22/84] fix --- .github/workflows/publish-command.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 1c74f63d4373..f5dafa16a39b 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -402,16 +402,16 @@ jobs: "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", }, { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "label": "${{ needs.start-publish-image-runner-2.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", }, { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "label": "${{ needs.start-publish-image-runner-3.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", }, { - "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", - "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + "label": "${{ needs.start-publish-image-runner-4.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", }, ] runs-on: ubuntu-latest From 5232e2185718a87d63bfc479b548e6d662db8b26 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:34:54 +0100 Subject: [PATCH 23/84] label test --- .github/workflows/publish-command.yml | 373 +++++++++++++------------- 1 file changed, 190 insertions(+), 183 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index f5dafa16a39b..12f73a5799a4 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -158,6 +158,13 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} label: airbyte-publisher-runner + manual-label-test: + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - id: label + run: | + ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} @@ -173,189 +180,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - publish-image: - timeout-minutes: 240 - needs: - - start-publish-image-runner-0 - - preprocess-matrix - strategy: - max-parallel: 5 - fail-fast: false - matrix: - connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: self-hosted - environment: more-secrets - steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - export_default_credentials: true - - name: Search for valid connector name format - id: regex - uses: AsasInnab/regex-action@v1 - with: - regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - regex_flags: "i" # required to be set for this plugin - search_string: ${{ matrix.connector }} - - name: Validate input workflow format - if: steps.regex.outputs.first_match != matrix.connector - run: echo "The connector provided has an invalid format!" && exit 1 - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - token: ${{ secrets.OCTAVIA_PAT }} - - name: Install Java - uses: actions/setup-java@v1 - with: - java-version: "17" - - name: Install Python - uses: actions/setup-python@v2 - with: - python-version: "3.9" - - name: Install Pyenv and Tox - run: | - python3 -m pip install --quiet virtualenv==16.7.9 --user - python3 -m virtualenv venv - source venv/bin/activate - pip install --quiet tox==3.24.4 - - name: Install yq - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - sudo add-apt-repository ppa:rmescandon/yq - sudo apt update - sudo apt install yq -y - - name: Test and install CI scripts - # all CI python packages have the prefix "ci_" - run: | - source venv/bin/activate - tox -r -c ./tools/tox_ci.ini - pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ matrix.connector }} - run: | - source venv/bin/activate - ci_credentials ${{ matrix.connector }} - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - - name: Set Name and Version Environment Vars - if: startsWith(matrix.connector, 'connectors') - run: | - source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - - name: Prepare Sentry - if: startsWith(matrix.connector, 'connectors') - run: | - curl -sL https://sentry.io/get-cli/ | bash - - name: Create Sentry Release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ matrix.connector }} - run: | - echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - id: publish - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - TZ: UTC - - name: Finalize Sentry release - if: startsWith(matrix.connector, 'connectors') - run: | - sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - env: - SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - SENTRY_ORG: airbyte-5j - SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{matrix.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{matrix.connector}} - - name: Check if connector in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - then exit 1 - fi - - name: Bump version in definitions yaml - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - connector="airbyte/${{ env.IMAGE_NAME }}" - definitionpath=./airbyte-config/init/src/main/resources/seed/ - sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - if [ -z "$sourcename" ] - then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - fi - - name: Run gradle process changes - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - ./gradlew :airbyte-config:init:processResources - - name: git config - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git config user.name 'Octavia Squidington III' - git config user.email 'octavia-squidington-iii@users.noreply.github.com' - - name: git commit and push - if: github.event.inputs.auto-bump-version == 'true' && success() - run: | - git add -u - git commit -m "auto-bump connector version" - git pull origin ${{ github.event.inputs.gitref }} - git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{matrix.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Couldn't auto-bump version for ${{matrix.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # publish-image: + # timeout-minutes: 240 + # needs: + # - start-publish-image-runner-0 + # - preprocess-matrix + # strategy: + # max-parallel: 5 + # fail-fast: false + # matrix: + # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + # runs-on: self-hosted + # environment: more-secrets + # steps: + # - name: Link comment to workflow run + # if: github.event.inputs.comment-id + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # - name: Set up Cloud SDK + # uses: google-github-actions/setup-gcloud@v0 + # with: + # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + # export_default_credentials: true + # - name: Search for valid connector name format + # id: regex + # uses: AsasInnab/regex-action@v1 + # with: + # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + # regex_flags: "i" # required to be set for this plugin + # search_string: ${{ matrix.connector }} + # - name: Validate input workflow format + # if: steps.regex.outputs.first_match != matrix.connector + # run: echo "The connector provided has an invalid format!" && exit 1 + # - name: Checkout Airbyte + # uses: actions/checkout@v2 + # with: + # repository: ${{ github.event.inputs.repo }} + # ref: ${{ github.event.inputs.gitref }} + # token: ${{ secrets.OCTAVIA_PAT }} + # - name: Install Java + # uses: actions/setup-java@v1 + # with: + # java-version: "17" + # - name: Install Python + # uses: actions/setup-python@v2 + # with: + # python-version: "3.9" + # - name: Install Pyenv and Tox + # run: | + # python3 -m pip install --quiet virtualenv==16.7.9 --user + # python3 -m virtualenv venv + # source venv/bin/activate + # pip install --quiet tox==3.24.4 + # - name: Install yq + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + # sudo add-apt-repository ppa:rmescandon/yq + # sudo apt update + # sudo apt install yq -y + # - name: Test and install CI scripts + # # all CI python packages have the prefix "ci_" + # run: | + # source venv/bin/activate + # tox -r -c ./tools/tox_ci.ini + # pip install --quiet -e ./tools/ci_* + # - name: Write Integration Test Credentials for ${{ matrix.connector }} + # run: | + # source venv/bin/activate + # ci_credentials ${{ matrix.connector }} + # env: + # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + # - name: Set Name and Version Environment Vars + # if: startsWith(matrix.connector, 'connectors') + # run: | + # source tools/lib/lib.sh + # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + # - name: Prepare Sentry + # if: startsWith(matrix.connector, 'connectors') + # run: | + # curl -sL https://sentry.io/get-cli/ | bash + # - name: Create Sentry Release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Publish ${{ matrix.connector }} + # run: | + # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + # id: publish + # env: + # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + # TZ: UTC + # - name: Finalize Sentry release + # if: startsWith(matrix.connector, 'connectors') + # run: | + # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + # env: + # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + # SENTRY_ORG: airbyte-5j + # SENTRY_PROJECT: airbyte-connectors + # - name: Add Published Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Successfully published ${{matrix.connector}} + # - name: Add Published Failure Comment + # if: github.event.inputs.comment-id && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Failed to publish ${{matrix.connector}} + # - name: Check if connector in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + # then exit 1 + # fi + # - name: Bump version in definitions yaml + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # connector="airbyte/${{ env.IMAGE_NAME }}" + # definitionpath=./airbyte-config/init/src/main/resources/seed/ + # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + # if [ -z "$sourcename" ] + # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + # fi + # - name: Run gradle process changes + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # ./gradlew :airbyte-config:init:processResources + # - name: git config + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git config user.name 'Octavia Squidington III' + # git config user.email 'octavia-squidington-iii@users.noreply.github.com' + # - name: git commit and push + # if: github.event.inputs.auto-bump-version == 'true' && success() + # run: | + # git add -u + # git commit -m "auto-bump connector version" + # git pull origin ${{ github.event.inputs.gitref }} + # git push origin ${{ github.event.inputs.gitref }} + # - name: Add Version Bump Success Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :rocket: Auto-bumped version for ${{matrix.connector}} + # - name: Add Version Bump Failure Comment + # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :x: Couldn't auto-bump version for ${{matrix.connector}} + # - name: Add Final Success Comment + # if: github.event.inputs.comment-id && success() + # uses: peter-evans/create-or-update-comment@v1 + # with: + # comment-id: ${{ github.event.inputs.comment-id }} + # body: | + # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs From 5d4a760085857d03c542d11c641e89d207316b66 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:48:44 +0100 Subject: [PATCH 24/84] couple more connector bumps --- .../connectors/destination-cassandra/Dockerfile | 2 +- .../connectors/destination-clickhouse-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-clickhouse/Dockerfile | 2 +- docs/integrations/destinations/clickhouse.md | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-cassandra/Dockerfile b/airbyte-integrations/connectors/destination-cassandra/Dockerfile index 2ee5c1d10a3b..5bb5b6b4dac1 100644 --- a/airbyte-integrations/connectors/destination-cassandra/Dockerfile +++ b/airbyte-integrations/connectors/destination-cassandra/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-cassandra COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-cassandra diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile index 2a88f7bf1248..8920d8d822e9 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile index a909fe50fab8..fa443a93f73f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/destination-clickhouse diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 4ad47c235121..2b3df385a8b2 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -79,6 +79,7 @@ Therefore, Airbyte ClickHouse destination will create tables and schemas using t | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:---------------------------------------------| +| 0.1.7 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | From 1cd9f993e1bfc36ed5d6f9b199dc70f87095e3fb Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 13:51:28 +0100 Subject: [PATCH 25/84] temp --- .github/workflows/publish-command.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 12f73a5799a4..c3bd7b0e2fd4 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -370,7 +370,7 @@ jobs: needs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - - publish-image # required to wait when the main job is done + # - publish-image # required to wait when the main job is done - find_valid_pat runs-on: ubuntu-latest steps: @@ -397,7 +397,7 @@ jobs: - start-publish-image-runner-3 - start-publish-image-runner-4 - preprocess-matrix - - publish-image # required to wait when the main job is done + # - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false From 5b9ac81e035f9b71a6537e616ccd0a0f9a323ca1 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 14:02:56 +0100 Subject: [PATCH 26/84] things --- .github/workflows/publish-command.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index c3bd7b0e2fd4..3517c01957b1 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -161,10 +161,14 @@ jobs: manual-label-test: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + # ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL steps: - id: label run: | - ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL + ls + ls ../ + pwd + cwd preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} From d43e431f2354f51323f9f622ab854addc946bf21 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 14:07:47 +0100 Subject: [PATCH 27/84] check this --- .github/workflows/publish-command.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 3517c01957b1..a67b96460535 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -165,10 +165,7 @@ jobs: steps: - id: label run: | - ls - ls ../ - pwd - cwd + ls /actions-runner/ preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} From 2a760fa89fe374e9367ab56ef2574f93e00f688b Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 14:13:47 +0100 Subject: [PATCH 28/84] lets gooo --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index a67b96460535..c48b3b7e791c 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -165,7 +165,7 @@ jobs: steps: - id: label run: | - ls /actions-runner/ + /actions-runner/config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},$GITHUB_RUN_ID-publisher preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} From 8a9e07454c5960f8b757fd4900c3aa7dc3327f66 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:01:07 +0100 Subject: [PATCH 29/84] more connectors --- .github/workflows/publish-command.yml | 383 +++++++++--------- .../connectors/destination-csv/Dockerfile | 2 +- .../destination-databricks/Dockerfile | 2 +- .../destination-dev-null/Dockerfile | 2 +- .../destination-dynamodb/Dockerfile | 2 +- docs/integrations/destinations/databricks.md | 1 + docs/integrations/destinations/dynamodb.md | 1 + 7 files changed, 191 insertions(+), 202 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index c48b3b7e791c..8dbfb7552ba3 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -69,7 +69,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-1: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 1 @@ -91,7 +90,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-2: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 2 @@ -113,7 +111,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-3: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 3 @@ -135,7 +132,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner start-publish-image-runner-4: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 4 @@ -157,15 +153,6 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner - manual-label-test: - needs: start-publish-image-runner-0 - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} - # ./config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},CUSTOMTESTLABEL - steps: - - id: label - run: | - /actions-runner/config.sh --labels ${{ needs.start-publish-image-runner-0.outputs.label }},$GITHUB_RUN_ID-publisher preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} @@ -181,189 +168,189 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - # publish-image: - # timeout-minutes: 240 - # needs: - # - start-publish-image-runner-0 - # - preprocess-matrix - # strategy: - # max-parallel: 5 - # fail-fast: false - # matrix: - # connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: self-hosted - # environment: more-secrets - # steps: - # - name: Link comment to workflow run - # if: github.event.inputs.comment-id - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - # - name: Set up Cloud SDK - # uses: google-github-actions/setup-gcloud@v0 - # with: - # service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} - # export_default_credentials: true - # - name: Search for valid connector name format - # id: regex - # uses: AsasInnab/regex-action@v1 - # with: - # regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" - # regex_flags: "i" # required to be set for this plugin - # search_string: ${{ matrix.connector }} - # - name: Validate input workflow format - # if: steps.regex.outputs.first_match != matrix.connector - # run: echo "The connector provided has an invalid format!" && exit 1 - # - name: Checkout Airbyte - # uses: actions/checkout@v2 - # with: - # repository: ${{ github.event.inputs.repo }} - # ref: ${{ github.event.inputs.gitref }} - # token: ${{ secrets.OCTAVIA_PAT }} - # - name: Install Java - # uses: actions/setup-java@v1 - # with: - # java-version: "17" - # - name: Install Python - # uses: actions/setup-python@v2 - # with: - # python-version: "3.9" - # - name: Install Pyenv and Tox - # run: | - # python3 -m pip install --quiet virtualenv==16.7.9 --user - # python3 -m virtualenv venv - # source venv/bin/activate - # pip install --quiet tox==3.24.4 - # - name: Install yq - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 - # sudo add-apt-repository ppa:rmescandon/yq - # sudo apt update - # sudo apt install yq -y - # - name: Test and install CI scripts - # # all CI python packages have the prefix "ci_" - # run: | - # source venv/bin/activate - # tox -r -c ./tools/tox_ci.ini - # pip install --quiet -e ./tools/ci_* - # - name: Write Integration Test Credentials for ${{ matrix.connector }} - # run: | - # source venv/bin/activate - # ci_credentials ${{ matrix.connector }} - # env: - # GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - # - name: Set Name and Version Environment Vars - # if: startsWith(matrix.connector, 'connectors') - # run: | - # source tools/lib/lib.sh - # DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile - # echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV - # echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - # - name: Prepare Sentry - # if: startsWith(matrix.connector, 'connectors') - # run: | - # curl -sL https://sentry.io/get-cli/ | bash - # - name: Create Sentry Release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Publish ${{ matrix.connector }} - # run: | - # echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - # ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache - # id: publish - # env: - # DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - # DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - # # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. - # TZ: UTC - # - name: Finalize Sentry release - # if: startsWith(matrix.connector, 'connectors') - # run: | - # sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" - # env: - # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} - # SENTRY_ORG: airbyte-5j - # SENTRY_PROJECT: airbyte-connectors - # - name: Add Published Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Successfully published ${{matrix.connector}} - # - name: Add Published Failure Comment - # if: github.event.inputs.comment-id && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Failed to publish ${{matrix.connector}} - # - name: Check if connector in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) - # destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) - # if [[ (-z "$sourcecheck" && -z "$destcheck") ]] - # then exit 1 - # fi - # - name: Bump version in definitions yaml - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # connector="airbyte/${{ env.IMAGE_NAME }}" - # definitionpath=./airbyte-config/init/src/main/resources/seed/ - # sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) - # destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) - # if [ -z "$sourcename" ] - # then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml - # else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml - # fi - # - name: Run gradle process changes - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # ./gradlew :airbyte-config:init:processResources - # - name: git config - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git config user.name 'Octavia Squidington III' - # git config user.email 'octavia-squidington-iii@users.noreply.github.com' - # - name: git commit and push - # if: github.event.inputs.auto-bump-version == 'true' && success() - # run: | - # git add -u - # git commit -m "auto-bump connector version" - # git pull origin ${{ github.event.inputs.gitref }} - # git push origin ${{ github.event.inputs.gitref }} - # - name: Add Version Bump Success Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :rocket: Auto-bumped version for ${{matrix.connector}} - # - name: Add Version Bump Failure Comment - # if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :x: Couldn't auto-bump version for ${{matrix.connector}} - # - name: Add Final Success Comment - # if: github.event.inputs.comment-id && success() - # uses: peter-evans/create-or-update-comment@v1 - # with: - # comment-id: ${{ github.event.inputs.comment-id }} - # body: | - # > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + publish-image: + timeout-minutes: 240 + needs: + - start-publish-image-runner-0 + - preprocess-matrix + strategy: + max-parallel: 5 + fail-fast: false + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that + environment: more-secrets + steps: + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }} + export_default_credentials: true + - name: Search for valid connector name format + id: regex + uses: AsasInnab/regex-action@v1 + with: + regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" + regex_flags: "i" # required to be set for this plugin + search_string: ${{ matrix.connector }} + - name: Validate input workflow format + if: steps.regex.outputs.first_match != matrix.connector + run: echo "The connector provided has an invalid format!" && exit 1 + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + token: ${{ secrets.OCTAVIA_PAT }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: "17" + - name: Install Python + uses: actions/setup-python@v2 + with: + python-version: "3.9" + - name: Install Pyenv and Tox + run: | + python3 -m pip install --quiet virtualenv==16.7.9 --user + python3 -m virtualenv venv + source venv/bin/activate + pip install --quiet tox==3.24.4 + - name: Install yq + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 + sudo add-apt-repository ppa:rmescandon/yq + sudo apt update + sudo apt install yq -y + - name: Test and install CI scripts + # all CI python packages have the prefix "ci_" + run: | + source venv/bin/activate + tox -r -c ./tools/tox_ci.ini + pip install --quiet -e ./tools/ci_* + - name: Write Integration Test Credentials for ${{ matrix.connector }} + run: | + source venv/bin/activate + ci_credentials ${{ matrix.connector }} + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + - name: Set Name and Version Environment Vars + if: startsWith(matrix.connector, 'connectors') + run: | + source tools/lib/lib.sh + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV + - name: Prepare Sentry + if: startsWith(matrix.connector, 'connectors') + run: | + curl -sL https://sentry.io/get-cli/ | bash + - name: Create Sentry Release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Publish ${{ matrix.connector }} + run: | + echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + id: publish + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. + TZ: UTC + - name: Finalize Sentry release + if: startsWith(matrix.connector, 'connectors') + run: | + sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" + env: + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} + SENTRY_ORG: airbyte-5j + SENTRY_PROJECT: airbyte-connectors + - name: Add Published Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Successfully published ${{matrix.connector}} + - name: Add Published Failure Comment + if: github.event.inputs.comment-id && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Failed to publish ${{matrix.connector}} + - name: Check if connector in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcecheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"source_definitions.yaml) + destcheck=$(yq e ".. | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\")" "$definitionpath"destination_definitions.yaml) + if [[ (-z "$sourcecheck" && -z "$destcheck") ]] + then exit 1 + fi + - name: Bump version in definitions yaml + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + connector="airbyte/${{ env.IMAGE_NAME }}" + definitionpath=./airbyte-config/init/src/main/resources/seed/ + sourcename=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"source_definitions.yaml) + destname=$(yq e ".[] | select(has(\"dockerRepository\")) | select(.dockerRepository == \"$connector\") | .name" "$definitionpath"destination_definitions.yaml) + if [ -z "$sourcename" ] + then yq e "(.[] | select(.name == \"$destname\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"destination_definitions.yaml + else yq e "(.[] | select(.name == \"$sourcename\").dockerImageTag)|=\"${{ env.IMAGE_VERSION }}\"" -i "$definitionpath"source_definitions.yaml + fi + - name: Run gradle process changes + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + ./gradlew :airbyte-config:init:processResources + - name: git config + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git config user.name 'Octavia Squidington III' + git config user.email 'octavia-squidington-iii@users.noreply.github.com' + - name: git commit and push + if: github.event.inputs.auto-bump-version == 'true' && success() + run: | + git add -u + git commit -m "auto-bump connector version" + git pull origin ${{ github.event.inputs.gitref }} + git push origin ${{ github.event.inputs.gitref }} + - name: Add Version Bump Success Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :rocket: Auto-bumped version for ${{matrix.connector}} + - name: Add Version Bump Failure Comment + if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :x: Couldn't auto-bump version for ${{matrix.connector}} + - name: Add Final Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -371,7 +358,7 @@ jobs: needs: - start-publish-image-runner-0 # required to get output from the start-runner job - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat runs-on: ubuntu-latest steps: @@ -398,7 +385,7 @@ jobs: - start-publish-image-runner-3 - start-publish-image-runner-4 - preprocess-matrix - # - publish-image # required to wait when the main job is done + - publish-image # required to wait when the main job is done - find_valid_pat strategy: fail-fast: false diff --git a/airbyte-integrations/connectors/destination-csv/Dockerfile b/airbyte-integrations/connectors/destination-csv/Dockerfile index 12791f0e238f..f9f3456bf150 100644 --- a/airbyte-integrations/connectors/destination-csv/Dockerfile +++ b/airbyte-integrations/connectors/destination-csv/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-csv COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/destination-csv diff --git a/airbyte-integrations/connectors/destination-databricks/Dockerfile b/airbyte-integrations/connectors/destination-databricks/Dockerfile index edad5c72a657..ae5588e07ac4 100644 --- a/airbyte-integrations/connectors/destination-databricks/Dockerfile +++ b/airbyte-integrations/connectors/destination-databricks/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-databricks COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/destination-databricks diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile index 007170ec3fa5..e37fc860dd67 100644 --- a/airbyte-integrations/connectors/destination-dev-null/Dockerfile +++ b/airbyte-integrations/connectors/destination-dev-null/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.6 +LABEL io.airbyte.version=0.2.7 LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile index 487b94174678..5f80d086a636 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-dynamodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-dynamodb diff --git a/docs/integrations/destinations/databricks.md b/docs/integrations/destinations/databricks.md index db105fd355bc..63635069f1af 100644 --- a/docs/integrations/destinations/databricks.md +++ b/docs/integrations/destinations/databricks.md @@ -104,6 +104,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.3 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.2.2 | 2022-06-13 | [\#13722](https://github.com/airbytehq/airbyte/pull/13722) | Rename to "Databricks Lakehouse". | | 0.2.1 | 2022-06-08 | [\#13630](https://github.com/airbytehq/airbyte/pull/13630) | Rename to "Databricks Delta Lake" and add field orders in the spec. | | 0.2.0 | 2022-05-15 | [\#12861](https://github.com/airbytehq/airbyte/pull/12861) | Use new public Databricks JDBC driver, and open source the connector. | diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md index 2934fb01e205..727218d51386 100644 --- a/docs/integrations/destinations/dynamodb.md +++ b/docs/integrations/destinations/dynamodb.md @@ -58,6 +58,7 @@ This connector by default uses 10 capacity units for both Read and Write in Dyna | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.4 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.3 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.2 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.1 | 2022-12-05 | [\#9314](https://github.com/airbytehq/airbyte/pull/9314) | Rename dynamo_db_table_name to dynamo_db_table_name_prefix. | From 24344174164b6d15e0346fc29449a56e37f78244 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:17:38 +0100 Subject: [PATCH 30/84] Delete TEMP-testing-command.yml --- .github/workflows/TEMP-testing-command.yml | 114 --------------------- 1 file changed, 114 deletions(-) delete mode 100644 .github/workflows/TEMP-testing-command.yml diff --git a/.github/workflows/TEMP-testing-command.yml b/.github/workflows/TEMP-testing-command.yml deleted file mode 100644 index d931ef501ee8..000000000000 --- a/.github/workflows/TEMP-testing-command.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: TEMP testing command -on: - workflow_dispatch: - inputs: - repo: - description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." - required: false - default: "airbytehq/airbyte" - gitref: - description: "The git ref to check out from the specified repository." - required: false - default: master - connector: - description: "Airbyte Connector" - required: true - run-tests: - description: "Should run tests when publishing" - required: true - default: "true" - comment-id: - description: "The comment-id of the slash command. Used to update the comment with the status." - required: false - auto-bump-version: - description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" - required: true - default: "true" - -jobs: - find_valid_pat: - name: "Find a PAT with room for actions" - timeout-minutes: 10 - runs-on: ubuntu-latest - outputs: - pat: ${{ steps.variables.outputs.pat }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - - name: Check PAT rate limits - id: variables - run: | - ./tools/bin/find_non_rate_limited_PAT \ - ${{ secrets.AIRBYTEIO_PAT }} \ - ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ - ${{ secrets.SUPERTOPHER_PAT }} \ - ${{ secrets.DAVINCHIA_PAT }} - ## Gradle Build - # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner: - name: Start Build EC2 Runner - strategy: - fail-fast: false - matrix: - ec2-instance-id: - [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", - ] - runs-on: ubuntu-latest - needs: find_valid_pat - outputs: - label: ${{ steps.start-ec2-runner.outputs.label }} - # ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v2 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} - - name: Start AWS Runner - id: start-ec2-runner - uses: ./.github/actions/start-aws-runner - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWCRET_ACCESS_KEY }} - github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: airbyte-publisher-runner - ec2-instance-id: ${{ matrix.ec2-instance-id }} - - stop-publish-image-runner: - name: Stop Build EC2 Runner - strategy: - fail-fast: false - matrix: - ec2-instance-id: - [ - "${{ github.run_id }}-${{ github.run_attempt }}-1", - "${{ github.run_id }}-${{ github.run_attempt }}-2", - "${{ github.run_id }}-${{ github.run_attempt }}-3", - "${{ github.run_id }}-${{ github.run_attempt }}-4", - "${{ github.run_id }}-${{ github.run_attempt }}-5 ", - ] - needs: - - start-publish-image-runner # required to get output from the start-runner job - # - publish-image # required to wait when the main job is done - - find_valid_pat - runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-2 - - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 - with: - mode: stop - github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner.outputs.label }} - ec2-instance-id: ${{ matrix.ec2-instance-id }} From 1748a5ec79251944679fd87cd9b744afc74f1f23 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 14:23:12 +0000 Subject: [PATCH 31/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index ea6afc10a8b6..5ba048fa3cf1 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -73,7 +73,7 @@ - name: Databricks Lakehouse destinationDefinitionId: 072d5540-f236-4294-ba7c-ade8fd918496 dockerRepository: airbyte/destination-databricks - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/databricks icon: databricks.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 81d82333f41e..3ee588ead68f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -996,7 +996,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-databricks:0.2.2" +- dockerImage: "airbyte/destination-databricks:0.2.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/databricks" connectionSpecification: From 79cb87d3fa848faf8954f3c9ca09e9688bfaf554 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 15:50:26 +0100 Subject: [PATCH 32/84] added comment describing bash part --- .github/workflows/publish-command.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 8dbfb7552ba3..9e81a42034f5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -159,6 +159,9 @@ jobs: outputs: connectorjson: ${{ steps.preprocess.outputs.connectorjson }} steps: + # given a string input of a single connector or comma separated list of connectors e.g. connector1, connector2 + # this step builds an array, by removing whitespace, add in quotation marks around connectors and braces [ ] at the start and end + # finally, it sets it as output from this job so we can use this array of connectors as our matrix strategy for publishing - id: preprocess run: | start="[\"" From 4fe526db618c9371c5c6fd36068895c803bfad2d Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:01:53 +0100 Subject: [PATCH 33/84] running single thread --- .github/workflows/publish-command.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 9e81a42034f5..5cfa5e01e468 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -181,7 +181,8 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that + # runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} environment: more-secrets steps: - name: Link comment to workflow run From bb7de7ad5ff48f4a079f3586393330752b810c24 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:05:35 +0100 Subject: [PATCH 34/84] catch sentry cli --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 5cfa5e01e468..1ee6fc4e0fd2 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -256,7 +256,7 @@ jobs: - name: Prepare Sentry if: startsWith(matrix.connector, 'connectors') run: | - curl -sL https://sentry.io/get-cli/ | bash + curl -sL https://sentry.io/get-cli/ | bash || echo "sentry cli already installed" - name: Create Sentry Release if: startsWith(matrix.connector, 'connectors') run: | From b690ac3d7a83832630d18314c04b89ec4fb40448 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 15:06:31 +0000 Subject: [PATCH 35/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index ea6afc10a8b6..b8f98b09dc29 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -67,7 +67,7 @@ - name: Clickhouse destinationDefinitionId: ce0d828e-1dc4-496c-b122-2da42e637e48 dockerRepository: airbyte/destination-clickhouse - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/destinations/clickhouse releaseStage: alpha - name: Databricks Lakehouse diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 81d82333f41e..1acd6b2c5eef 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -827,7 +827,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-clickhouse:0.1.6" +- dockerImage: "airbyte/destination-clickhouse:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/clickhouse" connectionSpecification: From 7e5866b6dc9206d160eab447938c56ac5136b3e4 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:38:53 +0100 Subject: [PATCH 36/84] destinations --- .../connectors/destination-e2e-test/Dockerfile | 2 +- .../connectors/destination-elasticsearch/Dockerfile | 2 +- airbyte-integrations/connectors/destination-gcs/Dockerfile | 2 +- airbyte-integrations/connectors/destination-jdbc/Dockerfile | 2 +- airbyte-integrations/connectors/destination-kafka/Dockerfile | 2 +- airbyte-integrations/connectors/destination-keen/Dockerfile | 2 +- airbyte-integrations/connectors/destination-kinesis/Dockerfile | 2 +- .../connectors/destination-local-json/Dockerfile | 2 +- .../connectors/destination-mariadb-columnstore/Dockerfile | 2 +- .../connectors/destination-meilisearch/Dockerfile | 2 +- .../connectors/destination-mongodb-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mongodb/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mqtt/Dockerfile | 2 +- .../connectors/destination-mssql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mssql/Dockerfile | 2 +- .../connectors/destination-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mysql/Dockerfile | 2 +- .../connectors/destination-oracle-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-oracle/Dockerfile | 2 +- .../connectors/destination-postgres-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-postgres/Dockerfile | 2 +- airbyte-integrations/connectors/destination-pubsub/Dockerfile | 2 +- airbyte-integrations/connectors/destination-pulsar/Dockerfile | 2 +- airbyte-integrations/connectors/destination-redis/Dockerfile | 2 +- .../connectors/destination-redshift/Dockerfile | 2 +- airbyte-integrations/connectors/destination-rockset/Dockerfile | 2 +- airbyte-integrations/connectors/destination-s3/Dockerfile | 2 +- airbyte-integrations/connectors/destination-scylla/Dockerfile | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- docs/integrations/destinations/e2e-test.md | 1 + docs/integrations/destinations/elasticsearch.md | 1 + docs/integrations/destinations/gcs.md | 1 + docs/integrations/destinations/kafka.md | 1 + docs/integrations/destinations/keen.md | 1 + docs/integrations/destinations/mariadb-columnstore.md | 1 + docs/integrations/destinations/meilisearch.md | 1 + docs/integrations/destinations/mongodb.md | 1 + docs/integrations/destinations/mqtt.md | 3 ++- docs/integrations/destinations/mssql.md | 1 + docs/integrations/destinations/mysql.md | 2 ++ docs/integrations/destinations/oracle.md | 1 + docs/integrations/destinations/postgres.md | 1 + docs/integrations/destinations/redshift.md | 1 + docs/integrations/destinations/rockset.md | 1 + docs/integrations/destinations/s3.md | 1 + 45 files changed, 47 insertions(+), 30 deletions(-) diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile index 4fc4ce7101b4..f0f8310b29b4 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/destination-e2e-test diff --git a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile index 1fe4f9d4a20d..6f8d3f8d7ba9 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-elasticsearch COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-elasticsearch diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index 4a234d7c6827..a486f174e069 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-gcs COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.7 +LABEL io.airbyte.version=0.2.8 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-jdbc/Dockerfile b/airbyte-integrations/connectors/destination-jdbc/Dockerfile index aa9e1177a2b1..a35e7fb7b3f2 100644 --- a/airbyte-integrations/connectors/destination-jdbc/Dockerfile +++ b/airbyte-integrations/connectors/destination-jdbc/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-jdbc COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.13 +LABEL io.airbyte.version=0.3.14 LABEL io.airbyte.name=airbyte/destination-jdbc diff --git a/airbyte-integrations/connectors/destination-kafka/Dockerfile b/airbyte-integrations/connectors/destination-kafka/Dockerfile index 39601850691d..11d6d7483afc 100644 --- a/airbyte-integrations/connectors/destination-kafka/Dockerfile +++ b/airbyte-integrations/connectors/destination-kafka/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-kafka COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/destination-kafka diff --git a/airbyte-integrations/connectors/destination-keen/Dockerfile b/airbyte-integrations/connectors/destination-keen/Dockerfile index 5b5c05b3c948..3d08ee9e10dd 100644 --- a/airbyte-integrations/connectors/destination-keen/Dockerfile +++ b/airbyte-integrations/connectors/destination-keen/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-keen COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/destination-keen diff --git a/airbyte-integrations/connectors/destination-kinesis/Dockerfile b/airbyte-integrations/connectors/destination-kinesis/Dockerfile index c3527bafc029..4513fa652bb8 100644 --- a/airbyte-integrations/connectors/destination-kinesis/Dockerfile +++ b/airbyte-integrations/connectors/destination-kinesis/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-kinesis COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-kinesis diff --git a/airbyte-integrations/connectors/destination-local-json/Dockerfile b/airbyte-integrations/connectors/destination-local-json/Dockerfile index 9ceea39cecdb..947616e66c03 100644 --- a/airbyte-integrations/connectors/destination-local-json/Dockerfile +++ b/airbyte-integrations/connectors/destination-local-json/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-local-json COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.10 +LABEL io.airbyte.version=0.2.11 LABEL io.airbyte.name=airbyte/destination-local-json diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile index d354e38ab27c..2a8d489ca116 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mariadb-columnstore COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-mariadb-columnstore diff --git a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile index 0abb9ddc620f..b664503d7ba5 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-meilisearch COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.12 +LABEL io.airbyte.version=0.2.13 LABEL io.airbyte.name=airbyte/destination-meilisearch diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile index 450beba6370f..74dab8dba94f 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mongodb/Dockerfile b/airbyte-integrations/connectors/destination-mongodb/Dockerfile index 4567fdca3a98..774a7d620a99 100644 --- a/airbyte-integrations/connectors/destination-mongodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-mongodb diff --git a/airbyte-integrations/connectors/destination-mqtt/Dockerfile b/airbyte-integrations/connectors/destination-mqtt/Dockerfile index 615836c81792..792c70786846 100644 --- a/airbyte-integrations/connectors/destination-mqtt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mqtt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mqtt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-mqtt diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile index f50b17ed7ab5..85280d2b9189 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile index c142424038bc..aa07ad06dba5 100644 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.19 +LABEL io.airbyte.version=0.1.20 LABEL io.airbyte.name=airbyte/destination-mssql diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile index ad65ba60ef06..d4503dcaddab 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index bc324b2bff11..29fa71d00ceb 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.20 LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile index 2270faca1ca6..f03271e99861 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 8ae09c1d1e30..7e2971d95735 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile index 7254fbcd0b6d..81a06278f614 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 67d9436e2fac..308263a7b563 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.20 +LABEL io.airbyte.version=0.3.21 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-pubsub/Dockerfile b/airbyte-integrations/connectors/destination-pubsub/Dockerfile index 7ea54f54929d..4bd1e25450c1 100644 --- a/airbyte-integrations/connectors/destination-pubsub/Dockerfile +++ b/airbyte-integrations/connectors/destination-pubsub/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-pubsub COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-pubsub diff --git a/airbyte-integrations/connectors/destination-pulsar/Dockerfile b/airbyte-integrations/connectors/destination-pulsar/Dockerfile index 809e5da68391..c96c68c6c3f2 100644 --- a/airbyte-integrations/connectors/destination-pulsar/Dockerfile +++ b/airbyte-integrations/connectors/destination-pulsar/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-pulsar COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-pulsar diff --git a/airbyte-integrations/connectors/destination-redis/Dockerfile b/airbyte-integrations/connectors/destination-redis/Dockerfile index 309a0d52e99f..c773173c33ab 100644 --- a/airbyte-integrations/connectors/destination-redis/Dockerfile +++ b/airbyte-integrations/connectors/destination-redis/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redis COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-redis diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile index 3e1528f9eaff..be77e3561248 100644 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ b/airbyte-integrations/connectors/destination-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.39 +LABEL io.airbyte.version=0.3.40 LABEL io.airbyte.name=airbyte/destination-redshift diff --git a/airbyte-integrations/connectors/destination-rockset/Dockerfile b/airbyte-integrations/connectors/destination-rockset/Dockerfile index 73477dc97bb6..136dbcd02b48 100644 --- a/airbyte-integrations/connectors/destination-rockset/Dockerfile +++ b/airbyte-integrations/connectors/destination-rockset/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-rockset COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-rockset diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 9400c975836b..0a09fb3112fc 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-s3 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.7 +LABEL io.airbyte.version=0.3.8 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-scylla/Dockerfile b/airbyte-integrations/connectors/destination-scylla/Dockerfile index 822b6c15c8f3..f7e349e0c1b1 100644 --- a/airbyte-integrations/connectors/destination-scylla/Dockerfile +++ b/airbyte-integrations/connectors/destination-scylla/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-scylla COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-scylla diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index dc1c4a0ba295..c424da73c199 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -20,5 +20,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=0.4.28 +LABEL io.airbyte.version=0.4.29 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/docs/integrations/destinations/e2e-test.md b/docs/integrations/destinations/e2e-test.md index 79eaf527420a..f00f5008bf4b 100644 --- a/docs/integrations/destinations/e2e-test.md +++ b/docs/integrations/destinations/e2e-test.md @@ -46,6 +46,7 @@ The OSS and Cloud variants have the same version number starting from version `0 | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--- | +| 0.2.4 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.2.2 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | | 0.2.1 | 2021-12-19 | [\#8824](https://github.com/airbytehq/airbyte/pull/8905) | Fix documentation URL. | diff --git a/docs/integrations/destinations/elasticsearch.md b/docs/integrations/destinations/elasticsearch.md index c2dfc2982b6a..b74fbe70441d 100644 --- a/docs/integrations/destinations/elasticsearch.md +++ b/docs/integrations/destinations/elasticsearch.md @@ -69,6 +69,7 @@ Enter the hostname and/or other configuration information ... | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.2 | 2022-04-19 | [11752](https://github.com/airbytehq/airbyte/pull/11752) | Reduce batch size to 32Mb | | 0.1.1 | 2022-02-10 | [10256](https://github.com/airbytehq/airbyte/pull/1256) | Add ExitOnOutOfMemoryError connectors | | 0.1.0 | 2021-10-13 | [7005](https://github.com/airbytehq/airbyte/pull/7005) | Initial release. | diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index d77d42dc776b..615258f1e53c 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -235,6 +235,7 @@ Under the hood, an Airbyte data stream in Json schema is first converted to an A | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.8 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | | 0.2.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.2.5 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | diff --git a/docs/integrations/destinations/kafka.md b/docs/integrations/destinations/kafka.md index 8f563c7e8972..4e70da2a3d43 100644 --- a/docs/integrations/destinations/kafka.md +++ b/docs/integrations/destinations/kafka.md @@ -98,6 +98,7 @@ _NOTE_: Some configurations for SSL are not available yet. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.7 | 2022-04-19 | [12134](https://github.com/airbytehq/airbyte/pull/12134) | Add PLAIN Auth | | 0.1.6 | 2022-02-15 | [10186](https://github.com/airbytehq/airbyte/pull/10186) | Add SCRAM-SHA-512 Auth | | 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/keen.md b/docs/integrations/destinations/keen.md index 598a0eff5168..55bc286c2459 100644 --- a/docs/integrations/destinations/keen.md +++ b/docs/integrations/destinations/keen.md @@ -78,6 +78,7 @@ If you have any questions, please reach out to us at team@keen.io and we’ll be | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.1 | 2021-12-30 | [\#8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | | 0.2.0 | 2021-09-10 | [\#5973](https://github.com/airbytehq/airbyte/pull/5973) | Fix timestamp inference for complex schemas | | 0.1.0 | 2021-08-18 | [\#5339](https://github.com/airbytehq/airbyte/pull/5339) | Keen Destination Release! | diff --git a/docs/integrations/destinations/mariadb-columnstore.md b/docs/integrations/destinations/mariadb-columnstore.md index c633ff91ce87..8a6a66e7daad 100644 --- a/docs/integrations/destinations/mariadb-columnstore.md +++ b/docs/integrations/destinations/mariadb-columnstore.md @@ -76,6 +76,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------| +| 0.1.6 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.5 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/meilisearch.md b/docs/integrations/destinations/meilisearch.md index 651812890d74..d8d6fef8a33a 100644 --- a/docs/integrations/destinations/meilisearch.md +++ b/docs/integrations/destinations/meilisearch.md @@ -33,5 +33,6 @@ The setup only requires two fields. First is the `host` which is the address at | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.2.11 | 2021-12-28 | [9156](https://github.com/airbytehq/airbyte/pull/9156) | Update connector fields title/description | diff --git a/docs/integrations/destinations/mongodb.md b/docs/integrations/destinations/mongodb.md index 42733f2b1a8a..297a45799ed6 100644 --- a/docs/integrations/destinations/mongodb.md +++ b/docs/integrations/destinations/mongodb.md @@ -93,6 +93,7 @@ Collection names should begin with an underscore or a letter character, and cann | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2021-12-30 | [8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | | 0.1.2 | 2021-10-18 | [6945](https://github.com/airbytehq/airbyte/pull/6945) | Create a secure-only MongoDb destination | diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md index 36e03c29a485..a6fd3140e772 100644 --- a/docs/integrations/destinations/mqtt.md +++ b/docs/integrations/destinations/mqtt.md @@ -82,4 +82,5 @@ _NOTE_: MQTT version 5 is not supported yet. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.2 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.1.1 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index a855693f3f74..f029e4762175 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -115,6 +115,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.19 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.18 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index fd2938f3626e..a4813991e5fb 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -110,6 +110,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.19 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.18 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | @@ -133,6 +134,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | +| 0.1.6 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.5 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 532d0fd3ea5f..03805f302ac8 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -92,6 +92,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.1.18 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.17 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.16 | 2022-04-06 | [11514](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index 97ad372c88c0..6e40332ec4fa 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -128,6 +128,7 @@ Now that you have set up the Postgres destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.3.21 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.20 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.3.19 | 2022-04-25 | [12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input | | 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index b53603ffb341..6df56b083c58 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -138,6 +138,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:------------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.40 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.39 | 2022-06-02 | [13415](https://github.com/airbytehq/airbyte/pull/13415) | Add dropdown to select Uploading Method.
**PLEASE NOTICE**: After this update your **uploading method** will be set to **Standard**, you will need to reconfigure the method to use **S3 Staging** again. | | 0.3.37 | 2022-05-23 | [13090](https://github.com/airbytehq/airbyte/pull/13090) | Removed redshiftDataTmpTableMode. Some refactoring. | | 0.3.36 | 2022-05-23 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | diff --git a/docs/integrations/destinations/rockset.md b/docs/integrations/destinations/rockset.md index 601bcd1be6ba..28f1ce653fc5 100644 --- a/docs/integrations/destinations/rockset.md +++ b/docs/integrations/destinations/rockset.md @@ -30,6 +30,7 @@ | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.2 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.1 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.0 | 2021-11-15 | [\#8006](https://github.com/airbytehq/airbyte/pull/8006) | Initial release| diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index cda1e5dfc1eb..77c62e1b97e8 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -315,6 +315,7 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:---------------------------------------------------------------------------------------------------------------------------| +| 0.3.8 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.7 | 2022-06-14 | [\#13483](https://github.com/airbytehq/airbyte/pull/13483) | Added support for int, long, float data types to Avro/Parquet formats. | | 0.3.6 | 2022-05-19 | [\#13043](https://github.com/airbytehq/airbyte/pull/13043) | Destination S3: Remove configurable part size. | | 0.3.5 | 2022-05-12 | [\#12797](https://github.com/airbytehq/airbyte/pull/12797) | Update spec to replace markdown. | From bb5820d2979edfa42c3297d466b23fce698c2a9d Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:39:10 +0100 Subject: [PATCH 37/84] + snowflake --- docs/integrations/destinations/snowflake.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index f3d3ede66f32..7c32aaa1f7fc 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -231,6 +231,7 @@ Now that you have set up the Snowflake destination connector, check out the foll | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.29 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.4.28 | 2022-05-18 | [\#12952](https://github.com/airbytehq/airbyte/pull/12952) | Apply buffering strategy on GCS staging | | 0.4.27 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.4.26 | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessages on error. | From 2d3a0de3475afb15c730c9d794a02941caeca655 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 16:39:57 +0100 Subject: [PATCH 38/84] saved --- docs/integrations/destinations/mssql.md | 1 + docs/integrations/destinations/oracle.md | 1 + docs/integrations/destinations/pubsub.md | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index f029e4762175..67481b38c77e 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -139,6 +139,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.8 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 03805f302ac8..96794b27e217 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -113,6 +113,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------|:-----------|:--------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------| +| 0.1.6 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.5 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/pubsub.md b/docs/integrations/destinations/pubsub.md index 7dbd9f55944d..804668afc2d6 100644 --- a/docs/integrations/destinations/pubsub.md +++ b/docs/integrations/destinations/pubsub.md @@ -89,6 +89,7 @@ Once you've configured PubSub as a destination, delete the Service Account Key f | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.4 | February 21, 2022 | [\#9819](https://github.com/airbytehq/airbyte/pull/9819) | Upgrade version of google-cloud-pubsub | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.2 | December 29, 2021 | [\#9183](https://github.com/airbytehq/airbyte/pull/9183) | Update connector fields title/description | From d7275bebead032e54a9777358711a0ce80d1fc47 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:02:50 +0000 Subject: [PATCH 39/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 4eb0728ee9bd..8d0f0f89ab7d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -53,7 +53,7 @@ - name: Cassandra destinationDefinitionId: 707456df-6f4f-4ced-b5c6-03f73bcad1c5 dockerRepository: airbyte/destination-cassandra - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/cassandra icon: cassandra.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 53ad5dfe9fca..27b44b1ce3a8 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -723,7 +723,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-cassandra:0.1.1" +- dockerImage: "airbyte/destination-cassandra:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/cassandra" connectionSpecification: From 8f08f967a3a86402f25873d478717b4fe83c2f59 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:06:51 +0000 Subject: [PATCH 40/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 8d0f0f89ab7d..0516ef225c03 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -141,7 +141,7 @@ - name: Local CSV destinationDefinitionId: 8be1cf83-fde1-477f-a4ad-318d23c9f3c6 dockerRepository: airbyte/destination-csv - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 documentationUrl: https://docs.airbyte.io/integrations/destinations/local-csv icon: file.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 27b44b1ce3a8..f53778d0f69a 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2308,7 +2308,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-csv:0.2.9" +- dockerImage: "airbyte/destination-csv:0.2.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/local-csv" connectionSpecification: From 0f243d8009a1ffa49c89f55d7b8606bf381b1aa9 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Fri, 17 Jun 2022 17:13:33 +0100 Subject: [PATCH 41/84] java source bumps --- airbyte-integrations/connectors/source-bigquery/Dockerfile | 2 +- .../connectors/source-clickhouse-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-clickhouse/Dockerfile | 2 +- .../connectors/source-cockroachdb-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-cockroachdb/Dockerfile | 2 +- .../connectors/source-db2-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-db2/Dockerfile | 2 +- .../connectors/source-e2e-test-cloud/Dockerfile | 2 +- airbyte-integrations/connectors/source-e2e-test/Dockerfile | 2 +- airbyte-integrations/connectors/source-jdbc/Dockerfile | 2 +- airbyte-integrations/connectors/source-kafka/Dockerfile | 2 +- .../connectors/source-mongodb-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mongodb-v2/Dockerfile | 2 +- .../connectors/source-mssql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mssql/Dockerfile | 2 +- .../connectors/source-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-mysql/Dockerfile | 2 +- .../connectors/source-oracle-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-oracle/Dockerfile | 2 +- .../connectors/source-postgres-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-postgres/Dockerfile | 2 +- airbyte-integrations/connectors/source-redshift/Dockerfile | 2 +- airbyte-integrations/connectors/source-relational-db/Dockerfile | 2 +- airbyte-integrations/connectors/source-sftp/Dockerfile | 2 +- airbyte-integrations/connectors/source-snowflake/Dockerfile | 2 +- airbyte-integrations/connectors/source-tidb/Dockerfile | 2 +- docs/integrations/sources/bigquery.md | 1 + docs/integrations/sources/clickhouse.md | 2 ++ docs/integrations/sources/cockroachdb.md | 2 ++ docs/integrations/sources/db2.md | 1 + docs/integrations/sources/e2e-test.md | 1 + docs/integrations/sources/kafka.md | 1 + docs/integrations/sources/mongodb-v2.md | 1 + docs/integrations/sources/mssql.md | 1 + docs/integrations/sources/mysql.md | 1 + docs/integrations/sources/oracle.md | 1 + docs/integrations/sources/postgres.md | 1 + docs/integrations/sources/redshift.md | 1 + docs/integrations/sources/sftp.md | 1 + docs/integrations/sources/snowflake.md | 1 + docs/integrations/sources/tidb.md | 1 + 41 files changed, 43 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/connectors/source-bigquery/Dockerfile b/airbyte-integrations/connectors/source-bigquery/Dockerfile index b1ea35f6ab86..dc501b8357f4 100644 --- a/airbyte-integrations/connectors/source-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/source-bigquery/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-bigquery COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-bigquery diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 8555fba96344..09e31f5ecfab 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index 1f9b1623150c..9f5553e81281 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile index f68558cfd607..b3ee12030a54 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index 8ae1e6922049..311ad518b9dc 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile index dfa81a644c1e..4340d6cc98bd 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index d2e3e152e7cc..379bf889370e 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile index c02dd323f6d6..0fbcce65c870 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.1.0 +LABEL io.airbyte.version=2.1.1 LABEL io.airbyte.name=airbyte/source-e2e-test-cloud diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index 0b2ff224a224..6d77c2b77af0 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.1.0 +LABEL io.airbyte.version=2.1.1 LABEL io.airbyte.name=airbyte/source-e2e-test diff --git a/airbyte-integrations/connectors/source-jdbc/Dockerfile b/airbyte-integrations/connectors/source-jdbc/Dockerfile index 8f44d096053e..cfb402f8f23c 100644 --- a/airbyte-integrations/connectors/source-jdbc/Dockerfile +++ b/airbyte-integrations/connectors/source-jdbc/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-jdbc COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.3 +LABEL io.airbyte.version=0.3.4 LABEL io.airbyte.name=airbyte/source-jdbc diff --git a/airbyte-integrations/connectors/source-kafka/Dockerfile b/airbyte-integrations/connectors/source-kafka/Dockerfile index b1283bd5a042..b34d30c35565 100644 --- a/airbyte-integrations/connectors/source-kafka/Dockerfile +++ b/airbyte-integrations/connectors/source-kafka/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-kafka COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.7 LABEL io.airbyte.name=airbyte/source-kafka diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile index f7abe1691c68..0c130eae19b1 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/source-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile index a84c6bcbd380..22d74754a13c 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile +++ b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mongodb-v2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.14 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index a4672e9759fd..ce584696e561 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.4.3 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index eadfd5211f0a..e52ba8240154 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.4.3 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 4a95c3c9cd04..27bf840a921f 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.5.10 +LABEL io.airbyte.version=0.5.12 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 1d321c00562d..0ae3f759028e 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.5.11 +LABEL io.airbyte.version=0.5.12 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index 96726701554a..030f9909887a 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -17,5 +17,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.15 +LABEL io.airbyte.version=0.3.16 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index cfc0c7aeaa39..38ec42a166de 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.15 +LABEL io.airbyte.version=0.3.16 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile index 6198e92f6087..608dcb4cc014 100644 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.25 +LABEL io.airbyte.version=0.4.26 LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile index 0daf7a6ea20c..ef066e80bb97 100644 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ b/airbyte-integrations/connectors/source-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.25 +LABEL io.airbyte.version=0.4.26 LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index a743455d2227..9a9c16a8ed54 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.10 +LABEL io.airbyte.version=0.3.11 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-relational-db/Dockerfile b/airbyte-integrations/connectors/source-relational-db/Dockerfile index 36baea09eafa..cba4be187f39 100644 --- a/airbyte-integrations/connectors/source-relational-db/Dockerfile +++ b/airbyte-integrations/connectors/source-relational-db/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-relational-db COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.2 LABEL io.airbyte.name=airbyte/source-relational-db diff --git a/airbyte-integrations/connectors/source-sftp/Dockerfile b/airbyte-integrations/connectors/source-sftp/Dockerfile index e71fcc42082e..4f1bf333ae73 100644 --- a/airbyte-integrations/connectors/source-sftp/Dockerfile +++ b/airbyte-integrations/connectors/source-sftp/Dockerfile @@ -14,5 +14,5 @@ ENV APPLICATION source-sftp COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-sftp diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 7d904bc10687..7a917c353705 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.12 +LABEL io.airbyte.version=0.1.13 LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index 6179f1f2b654..d322630a76e5 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/docs/integrations/sources/bigquery.md b/docs/integrations/sources/bigquery.md index dae4be2c1df8..a2c411b55aee 100644 --- a/docs/integrations/sources/bigquery.md +++ b/docs/integrations/sources/bigquery.md @@ -88,6 +88,7 @@ Once you've configured BigQuery as a source, delete the Service Account Key from | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.8 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.7 | 2022-04-11 | [11484](https://github.com/airbytehq/airbyte/pull/11484) | BigQuery connector escape column names | | 0.1.6 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.5 | 2021-12-23 | [8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 38e2e2295131..b73a8e1e4257 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -78,6 +78,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| +| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.10 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.9 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | | 0.1.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | @@ -93,6 +94,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.7 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | | 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2021-12-29 | [\#9182](https://github.com/airbytehq/airbyte/pull/9182) [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY. Fixed tests | diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index 3de1b54f3b0f..df7d0f1bb923 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,6 +95,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- | :--- | +| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | @@ -111,6 +112,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.8 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index 7a029ecb619e..d1346312ec10 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -62,6 +62,7 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index 01766784682e..dcbcd8c2ca4d 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -63,6 +63,7 @@ The OSS and Cloud variants have the same version number. The Cloud variant was i | Version | Date | Pull request | Notes | | --- | --- | --- | --- | +| 2.1.1 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 2.1.0 | 2021-02-12 | [\#10298](https://github.com/airbytehq/airbyte/pull/10298) | Support stream duplication to quickly create a multi-stream catalog. | | 2.0.0 | 2021-02-01 | [\#9954](https://github.com/airbytehq/airbyte/pull/9954) | Remove legacy modes. Use more efficient Json generator. | | 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | diff --git a/docs/integrations/sources/kafka.md b/docs/integrations/sources/kafka.md index b50961fbdcb2..48815e18389f 100644 --- a/docs/integrations/sources/kafka.md +++ b/docs/integrations/sources/kafka.md @@ -44,6 +44,7 @@ The Kafka source connector supports the following[sync modes](https://docs.airby | Version | Date | Pull Request | Subject | | :------ | :-------- | :------------------------------------------------------| :---------------------------------------- | +| 0.1.7 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.6 | 2022-05-29 | [12903](https://github.com/airbytehq/airbyte/pull/12903) | Add Polling Time to Specification (default 100 ms) | | 0.1.5 | 2022-04-19 | [12134](https://github.com/airbytehq/airbyte/pull/12134) | Add PLAIN Auth | | 0.1.4 | 2022-02-15 | [10186](https://github.com/airbytehq/airbyte/pull/10186) | Add SCRAM-SHA-512 Auth | diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index 3b7feb848676..df3d7f079206 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -102,6 +102,7 @@ For more information regarding configuration parameters, please see [MongoDb Doc | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.15 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.14 | 2022-05-05 | [12428](https://github.com/airbytehq/airbyte/pull/12428) | JsonSchema: Add properties to fields with type 'object' | | 0.1.13 | 2022-02-21 | [10276](https://github.com/airbytehq/airbyte/pull/10276) | Create a custom codec registry to handle DBRef MongoDB objects | | 0.1.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 203b6ddc004b..0801be8cacc1 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -302,6 +302,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------------------------------------------------- | :------------------------------------- | +| 0.4.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.4.2 | 2022-06-06 | [13435](https://github.com/airbytehq/airbyte/pull/13435) | Adjust JDBC fetch size based on max memory and max row size | | 0.4.1 | 2022-05-25 | [13419](https://github.com/airbytehq/airbyte/pull/13419) | Correct enum for Standard method. | | 0.4.0 | 2022-05-25 | [12759](https://github.com/airbytehq/airbyte/pull/12759) [13168](https://github.com/airbytehq/airbyte/pull/13168) | For CDC, Add option to ignore existing data and only sync new changes from the database. | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index ba906bd50b1a..906fa8d71021 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -185,6 +185,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| +| 0.5.12 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.5.11 | 2022-05-03 | [12544](https://github.com/airbytehq/airbyte/pull/12544) | Prevent source from hanging under certain circumstances by adding a watcher for orphaned threads. | | 0.5.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.5.9 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 031e55def8e2..83066723baca 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,6 +132,7 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:------------------------------------------------| +| 0.3.16 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.15 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.3.14 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.3.13 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 068c75a31b89..9d4338cdbf07 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -295,6 +295,7 @@ One optimization on the Airbyte side is to break one large and long sync into mu | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.4.26 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.4.25 | 2022-06-15 | [13823](https://github.com/airbytehq/airbyte/pull/13823) | Publish adaptive postgres source that enforces ssl on cloud | | 0.4.24 | 2022-06-14 | [13549](https://github.com/airbytehq/airbyte/pull/13549) | Fixed truncated precision if the value of microseconds or seconds is 0 | | 0.4.23 | 2022-06-13 | [13655](https://github.com/airbytehq/airbyte/pull/13745) | Fixed handling datetime cursors when upgrading from older versions of the connector | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index 33bd04918bc2..6d06cc483574 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -54,6 +54,7 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.3.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |0 | 0.3.9 | 2022-02-21 | [9744](https://github.com/airbytehq/airbyte/pull/9744) | List only the tables on which the user has SELECT permissions. | 0.3.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/sources/sftp.md b/docs/integrations/sources/sftp.md index 5e5fd6f4aef0..ae40e624b5bb 100644 --- a/docs/integrations/sources/sftp.md +++ b/docs/integrations/sources/sftp.md @@ -57,4 +57,5 @@ More formats \(e.g. Apache Avro\) will be supported in the future. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-------------|:----------------| +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.0 | 2021-24-05 | | Initial version | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index ddeca9ac1d03..476163608652 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -103,6 +103,7 @@ Field | Description | | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-27 | [10953](https://github.com/airbytehq/airbyte/pull/10953) | Implement OAuth flow | | 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 5cea0d66f733..4b82a1728354 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -120,5 +120,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | | :------ | :--- | :----------- | ------- | +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | From d409ed6ff3cfc61c5e85863fb97044dbc344fa36 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:23:02 +0000 Subject: [PATCH 42/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 0516ef225c03..e9ec57e3aea7 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -80,7 +80,7 @@ - name: DynamoDB destinationDefinitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 dockerRepository: airbyte/destination-dynamodb - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/dynamodb icon: dynamodb.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index f53778d0f69a..536db5d8560f 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1160,7 +1160,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-dynamodb:0.1.3" +- dockerImage: "airbyte/destination-dynamodb:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/dynamodb" connectionSpecification: From 8ad00425b36cb543780ab3236682da4aec868766 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:26:58 +0000 Subject: [PATCH 43/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e9ec57e3aea7..f2c733ba56b6 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -87,7 +87,7 @@ - name: E2E Testing destinationDefinitionId: 2eb65e87-983a-4fd7-b3e3-9d9dc6eb8537 dockerRepository: airbyte/destination-e2e-test - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/destinations/e2e-test icon: airbyte.svg - destinationDefinitionId: 68f351a7-2745-4bef-ad7f-996b8e51bb8c diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 536db5d8560f..5ffd35e2eb8e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1241,7 +1241,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-e2e-test:0.2.2" +- dockerImage: "airbyte/destination-e2e-test:0.2.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/e2e-test" connectionSpecification: From fc91f0c73fc3d407a5d41b64dd82c5e29721989a Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:34:17 +0000 Subject: [PATCH 44/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 05860d916e33..3c11ae33904e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -96,7 +96,7 @@ - name: BigQuery sourceDefinitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c dockerRepository: airbyte/source-bigquery - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/bigquery icon: bigquery.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index aa913b0ff5d9..9a857ec50e35 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -830,7 +830,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bigquery:0.1.7" +- dockerImage: "airbyte/source-bigquery:0.1.8" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bigquery" connectionSpecification: From 73b3f6666a8e53ba421b83744af9e9839a68cfda Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:46:59 +0000 Subject: [PATCH 45/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index f2c733ba56b6..aec4cd1fdafb 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -127,7 +127,7 @@ - name: Kafka destinationDefinitionId: 9f760101-60ae-462f-9ee6-b7a9dafd454d dockerRepository: airbyte/destination-kafka - dockerImageTag: 0.1.8 + dockerImageTag: 0.1.9 documentationUrl: https://docs.airbyte.io/integrations/destinations/kafka icon: kafka.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 5ffd35e2eb8e..f1245570662e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1959,7 +1959,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-kafka:0.1.8" +- dockerImage: "airbyte/destination-kafka:0.1.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/kafka" connectionSpecification: From 83fe86a722102a732d89a2ad7149c55d91eca9b5 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 16:54:12 +0000 Subject: [PATCH 46/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index aec4cd1fdafb..38647c5f0763 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -60,7 +60,7 @@ - name: Chargify (Keen) destinationDefinitionId: 81740ce8-d764-4ea7-94df-16bb41de36ae dockerRepository: airbyte/destination-keen - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/keen icon: chargify.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index f1245570662e..55f28aa74ef7 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -788,7 +788,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-keen:0.2.2" +- dockerImage: "airbyte/destination-keen:0.2.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/keen" connectionSpecification: From 5f8af97d9b0bcd222e388ed1178e40638f0355a7 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 17:03:10 +0000 Subject: [PATCH 47/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 38647c5f0763..53e12cbe7e2c 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -134,7 +134,7 @@ - name: Kinesis destinationDefinitionId: 6d1d66d4-26ab-4602-8d32-f85894b04955 dockerRepository: airbyte/destination-kinesis - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/kinesis icon: kinesis.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 55f28aa74ef7..b896cb3c0180 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2250,7 +2250,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-kinesis:0.1.2" +- dockerImage: "airbyte/destination-kinesis:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/kinesis" connectionSpecification: From 4927fe92d9dcc06fe461230a8b4654b449ad23e6 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 17:51:31 +0000 Subject: [PATCH 48/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 3c11ae33904e..fe0e8a786045 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -398,7 +398,7 @@ - name: IBM Db2 sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 dockerRepository: airbyte/source-db2 - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/db2 icon: db2.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9a857ec50e35..8760dfe4554b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3734,7 +3734,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-db2:0.1.10" +- dockerImage: "airbyte/source-db2:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/db2" connectionSpecification: From bc573eb47950b118c500c8fe27fe70675ce1fcac Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 18:18:56 +0000 Subject: [PATCH 49/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index fe0e8a786045..c12576f4f8fc 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -224,7 +224,7 @@ - name: E2E Testing sourceDefinitionId: d53f9084-fa6b-4a5a-976c-5b8392f4ad8a dockerRepository: airbyte/source-e2e-test - dockerImageTag: 2.1.0 + dockerImageTag: 2.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/e2e-test icon: airbyte.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8760dfe4554b..53c8051b8537 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1691,7 +1691,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-e2e-test:2.1.0" +- dockerImage: "airbyte/source-e2e-test:2.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/e2e-test" connectionSpecification: From c9a8248f9263b6505693adbe15ede62475f2b914 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 18:33:04 +0000 Subject: [PATCH 50/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c12576f4f8fc..327e1d4bd599 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -446,7 +446,7 @@ - name: Kafka sourceDefinitionId: d917a47b-8537-4d0d-8c10-36a9928d4265 dockerRepository: airbyte/source-kafka - dockerImageTag: 0.1.6 + dockerImageTag: 0.1.7 documentationUrl: https://docs.airbyte.io/integrations/sources/kafka icon: kafka.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 53c8051b8537..9b4988cca23d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4057,7 +4057,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-kafka:0.1.6" +- dockerImage: "airbyte/source-kafka:0.1.7" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/kafka" connectionSpecification: @@ -4148,6 +4148,11 @@ \ and returns them incrementally from each poll." type: "integer" default: 500 + polling_time: + title: "Polling Time" + description: "Amount of time Kafka connector should try to poll for messages." + type: "integer" + default: 100 protocol: title: "Protocol" type: "object" From e89cf1d734b9038ed4590f6886e68a29a45235f2 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 18:40:51 +0000 Subject: [PATCH 51/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 327e1d4bd599..fddec0d3c39e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -557,7 +557,7 @@ - name: MongoDb sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e dockerRepository: airbyte/source-mongodb-v2 - dockerImageTag: 0.1.14 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/sources/mongodb-v2 icon: mongodb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9b4988cca23d..9ea1f1eeee91 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5454,7 +5454,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-mongodb-v2:0.1.14" +- dockerImage: "airbyte/source-mongodb-v2:0.1.15" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2" changelogUrl: "https://docs.airbyte.io/integrations/sources/mongodb-v2" From cca4264e942c382a3ca1380caa7ee489389176a5 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 19:21:38 +0000 Subject: [PATCH 52/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index fddec0d3c39e..5c80adce1744 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -573,7 +573,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.5.11 + dockerImageTag: 0.5.12 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9ea1f1eeee91..e9ae15fa163e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5618,7 +5618,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.5.11" +- dockerImage: "airbyte/source-mysql:0.5.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: From 0c3bb2a2c835cf8d5bd7ae59809792cd67682bbd Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 20:15:19 +0000 Subject: [PATCH 53/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 53e12cbe7e2c..139e49082d4a 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -277,7 +277,7 @@ - name: MariaDB ColumnStore destinationDefinitionId: 294a4790-429b-40ae-9516-49826b9702e1 dockerRepository: airbyte/destination-mariadb-columnstore - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/destinations/mariadb-columnstore icon: mariadb.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index b896cb3c0180..70056999c850 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -4769,7 +4769,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.4" +- dockerImage: "airbyte/destination-mariadb-columnstore:0.1.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mariadb-columnstore" connectionSpecification: From 51b476e3849c7d8717907396ebaf5ea86b0cc934 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 20:19:52 +0000 Subject: [PATCH 54/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 139e49082d4a..e88f61bc2652 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -169,7 +169,7 @@ - name: MeiliSearch destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e dockerRepository: airbyte/destination-meilisearch - dockerImageTag: 0.2.12 + dockerImageTag: 0.2.13 documentationUrl: https://docs.airbyte.io/integrations/destinations/meilisearch icon: meilisearch.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 70056999c850..8b048277390a 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2692,7 +2692,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-meilisearch:0.2.12" +- dockerImage: "airbyte/destination-meilisearch:0.2.13" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/meilisearch" connectionSpecification: From 5a758125f67c6cef4296c498721f67243a8bf107 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 21:25:06 +0000 Subject: [PATCH 55/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5c80adce1744..1e57bd97bfef 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -715,7 +715,7 @@ - name: Postgres sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 dockerRepository: airbyte/source-postgres - dockerImageTag: 0.4.25 + dockerImageTag: 0.4.26 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e9ae15fa163e..2621cef824fe 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6724,7 +6724,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-postgres:0.4.25" +- dockerImage: "airbyte/source-postgres:0.4.26" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" connectionSpecification: From f76c6f84fd0e66e659eb9d508e1cf82bae0bc75b Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 17 Jun 2022 21:40:13 +0000 Subject: [PATCH 56/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1e57bd97bfef..2ac9f76f6033 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1061,7 +1061,7 @@ - name: SFTP sourceDefinitionId: a827c52e-791c-4135-a245-e233c5255199 dockerRepository: airbyte/source-sftp - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.com/integrations/sources/sftp sourceType: file releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 2621cef824fe..89ccf1c9506b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10123,7 +10123,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-sftp:0.1.1" +- dockerImage: "airbyte/source-sftp:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/source/sftp" connectionSpecification: From 32e41e558b8dfc2dec7b1a364158c882b05dc0af Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sat, 18 Jun 2022 09:57:44 +0000 Subject: [PATCH 57/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e88f61bc2652..9a871e404bbf 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -120,7 +120,7 @@ - name: Google PubSub destinationDefinitionId: 356668e2-7e34-47f3-a3b0-67a8a481b692 dockerRepository: airbyte/destination-pubsub - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/destinations/pubsub icon: googlepubsub.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 8b048277390a..8e420192c45e 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -1926,7 +1926,7 @@ supported_destination_sync_modes: - "append" - "overwrite" -- dockerImage: "airbyte/destination-pubsub:0.1.4" +- dockerImage: "airbyte/destination-pubsub:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/pubsub" connectionSpecification: From 275a4e7cfe5185cc6c195f5ef497a79f04fd8c19 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sat, 18 Jun 2022 10:19:30 +0000 Subject: [PATCH 58/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 9a871e404bbf..c4bfb6a03c58 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -204,7 +204,7 @@ - name: Pulsar destinationDefinitionId: 2340cbba-358e-11ec-8d3d-0242ac130203 dockerRepository: airbyte/destination-pulsar - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/pulsar icon: pulsar.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index 8e420192c45e..d83c8670dd39 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3433,7 +3433,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-pulsar:0.1.1" +- dockerImage: "airbyte/destination-pulsar:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/pulsar" connectionSpecification: From e9e05834b30b723edf639c4985fabe8360ff20e8 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sat, 18 Jun 2022 10:23:39 +0000 Subject: [PATCH 59/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index c4bfb6a03c58..1e233399275d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -218,7 +218,7 @@ - name: Redis destinationDefinitionId: d4d3fef9-e319-45c2-881a-bd02ce44cc9f dockerRepository: airbyte/destination-redis - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/redis icon: redis.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index d83c8670dd39..af180a6ce9aa 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3623,7 +3623,7 @@ supportsDBT: false supported_destination_sync_modes: - "append" -- dockerImage: "airbyte/destination-redis:0.1.1" +- dockerImage: "airbyte/destination-redis:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/redis" connectionSpecification: From c7bb0562a05d230d5301135f3de86cceb70f29f7 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Sat, 18 Jun 2022 11:38:44 +0000 Subject: [PATCH 60/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 1e233399275d..d0a18f0cbb63 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -291,7 +291,7 @@ - name: Scylla destinationDefinitionId: 3dc6f384-cd6b-4be3-ad16-a41450899bf0 dockerRepository: airbyte/destination-scylla - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/scylla icon: scylla.svg - name: Google Sheets diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index af180a6ce9aa..5134c2cd9bf1 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -4948,7 +4948,7 @@ supported_destination_sync_modes: - "append" - "append_dedup" -- dockerImage: "airbyte/destination-scylla:0.1.1" +- dockerImage: "airbyte/destination-scylla:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/scylla" connectionSpecification: From e2d7624ac137059b4b1bc7157fd04dab54642882 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 14:25:39 +0100 Subject: [PATCH 61/84] remove twice-defined methods --- .../MongodbDestinationAcceptanceTest.java | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index 2115990996e8..49dcaadfa742 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -111,26 +111,6 @@ protected void tearDown(final TestDestinationEnv testEnv) { container.close(); } - @Override - protected TestDataComparator getTestDataComparator() { - return new AdvancedTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - /* Helpers */ private JsonNode getAuthTypeConfig() { From 05121d3eff01c207be13dae841e5167be9e0f042 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 16:51:56 +0100 Subject: [PATCH 62/84] label things --- .github/actions/start-aws-runner/action.yml | 3 ++- .github/workflows/publish-command.yml | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 7f268783fc60..b6e16b0151a8 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -41,7 +41,7 @@ runs: aws-region: us-east-2 - name: Start EC2 runner id: start-ec2-runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: start github-token: ${{ inputs.github-token }} @@ -55,3 +55,4 @@ runs: {"Key": "Repository", "Value": "${{ github.repository }}"}, {"Key": "Branch", "Value": "${{ github.ref }}"} ] + label: ${{ inputs.label }} diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 1ee6fc4e0fd2..ba991515ee1f 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -69,6 +69,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher start-publish-image-runner-1: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 1 @@ -90,6 +91,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher start-publish-image-runner-2: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 2 @@ -111,6 +113,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher start-publish-image-runner-3: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 3 @@ -132,6 +135,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher start-publish-image-runner-4: if: github.event.inputs.parallel == 'true' && success() name: Start Build EC2 Runner 4 @@ -153,6 +157,7 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher preprocess-matrix: needs: start-publish-image-runner-0 runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} @@ -182,7 +187,7 @@ jobs: matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} # runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + runs-on: ${{ github.run_id }}-publisher environment: more-secrets steps: - name: Link comment to workflow run From 9106fed5294803ee1a8cadcb23a73796e5834c66 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 17:01:29 +0100 Subject: [PATCH 63/84] revert action --- .github/actions/start-aws-runner/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index b6e16b0151a8..7f268783fc60 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -41,7 +41,7 @@ runs: aws-region: us-east-2 - name: Start EC2 runner id: start-ec2-runner - uses: airbytehq/ec2-github-runner@base64v1.1.0 + uses: supertopher/ec2-github-runner@base64v1.0.10 with: mode: start github-token: ${{ inputs.github-token }} @@ -55,4 +55,3 @@ runs: {"Key": "Repository", "Value": "${{ github.repository }}"}, {"Key": "Branch", "Value": "${{ github.ref }}"} ] - label: ${{ inputs.label }} From be6ce9df349215ab2ffa61e5c3ab190283768922 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 17:19:06 +0100 Subject: [PATCH 64/84] using the new test action --- .github/workflows/publish-command.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index ba991515ee1f..cc67cd9f0383 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -64,7 +64,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner + uses: ./.github/actions/start-aws-runner-phlairTest with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -86,7 +86,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner + uses: ./.github/actions/start-aws-runner-phlairTest with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -108,7 +108,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner + uses: ./.github/actions/start-aws-runner-phlairTest with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -130,7 +130,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner + uses: ./.github/actions/start-aws-runner-phlairTest with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -152,7 +152,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner + uses: ./.github/actions/start-aws-runner-phlairTest with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} From 5597760842eafc527a7f3de09bd5ad6d6e84c359 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 18:07:44 +0100 Subject: [PATCH 65/84] point at action --- .github/actions/start-aws-runner/action.yml | 3 ++- .github/workflows/publish-command.yml | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 7f268783fc60..55480832fdfb 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -41,7 +41,7 @@ runs: aws-region: us-east-2 - name: Start EC2 runner id: start-ec2-runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.0.10 with: mode: start github-token: ${{ inputs.github-token }} @@ -49,6 +49,7 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} + label: ${{ github.run_id }}-publisher aws-resource-tags: > [ {"Key": "BuildType", "Value": "oss"}, diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index cc67cd9f0383..ba991515ee1f 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -64,7 +64,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner-phlairTest + uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -86,7 +86,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner-phlairTest + uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -108,7 +108,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner-phlairTest + uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -130,7 +130,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner-phlairTest + uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} @@ -152,7 +152,7 @@ jobs: ref: ${{ github.event.inputs.gitref }} - name: Start AWS Runner id: start-ec2-runner - uses: ./.github/actions/start-aws-runner-phlairTest + uses: ./.github/actions/start-aws-runner with: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} From 55c3b644ced06b6e67be768750c193d81d2ba1ab Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 18:10:51 +0100 Subject: [PATCH 66/84] wrong tag on action --- .github/actions/start-aws-runner/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 55480832fdfb..21d81f1c0aaf 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -41,7 +41,7 @@ runs: aws-region: us-east-2 - name: Start EC2 runner id: start-ec2-runner - uses: airbytehq/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: start github-token: ${{ inputs.github-token }} From 8e6bc8d0b45eae06e9c042ef846ce6ad586d8992 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 18:56:10 +0100 Subject: [PATCH 67/84] update pool label --- .github/actions/start-aws-runner/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 21d81f1c0aaf..14ad2239afdc 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -49,7 +49,7 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} - label: ${{ github.run_id }}-publisher + label: publisher-pool-${{ github.run_id }} aws-resource-tags: > [ {"Key": "BuildType", "Value": "oss"}, From 82f2d9738ac62f8416a4193c880765c9d4772b14 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 19:01:06 +0100 Subject: [PATCH 68/84] update to use new ec2-github-runner fork --- .github/workflows/publish-command.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index ba991515ee1f..b047c393f8f5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -378,7 +378,7 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} @@ -427,7 +427,7 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} From 72f934be61d9b67a9a3a9127e89cb979e6520d49 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 19:14:27 +0100 Subject: [PATCH 69/84] this needs to be more generic than publisher --- .github/actions/start-aws-runner/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 14ad2239afdc..896a06d30e47 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -49,7 +49,7 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} - label: publisher-pool-${{ github.run_id }} + label: runner-pool-${{ github.run_id }} aws-resource-tags: > [ {"Key": "BuildType", "Value": "oss"}, From e855cb688ccc9c4c5796e9366fe3f7158f70deb7 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 19:20:14 +0100 Subject: [PATCH 70/84] change publish to run on pool --- .github/workflows/publish-command.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index b047c393f8f5..533a9e8c8cbf 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -186,8 +186,7 @@ jobs: fail-fast: false matrix: connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} - # runs-on: self-hosted # TODO: need to change this to run on custom label when I can implement that - runs-on: ${{ github.run_id }}-publisher + runs-on: runner-pool-${{ github.run_id }} environment: more-secrets steps: - name: Link comment to workflow run From 036565af909f51ed9bf13580a3ae745b17984ebb Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 19:33:14 +0100 Subject: [PATCH 71/84] add comment about runner-pool usage --- .github/actions/start-aws-runner/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 896a06d30e47..c3b94df610b6 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -49,6 +49,8 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} + # this adds a label to group any EC2 runners spun up within the same action run + # this enables creating a pool of runners to run multiple/matrix jobs on in parallel label: runner-pool-${{ github.run_id }} aws-resource-tags: > [ From 4f2d60e3c6ffba68f6831e060a9a248a934d7e39 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Mon, 20 Jun 2022 19:40:26 +0100 Subject: [PATCH 72/84] updated publish command docs for multi & parallel connector runs --- docs/connector-development/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index da91130ca49a..ca18ddc8f81e 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -136,12 +136,13 @@ Once you've finished iterating on the changes to a connector as specified in its ### The /publish command Publishing a connector can be done using the `/publish` command as outlined in the above section. The command runs a [github workflow](https://github.com/airbytehq/airbyte/actions/workflows/publish-command.yml), and has the following configurable parameters: -* **connector** - Required. This tells the workflow which connector to publish. e.g. `connector=connectors/source-amazon-ads` +* **connector** - Required. This tells the workflow which connector to publish. e.g. `connector=connectors/source-amazon-ads`. This can also be a comma-separated list of many connectors, e.g. `connector=connectors/source-s3,connectors/destination-postgres,connectors/source-facebook-marketing`. See the parallel flag below if publishing multiple connectors. * **repo** - Defaults to the main airbyte repo. Set this when building connectors from forked repos. e.g. `repo=userfork/airbyte` * **gitref** - Defaults to the branch of the PR where the /publish command is run as a comment. If running manually, set this to your branch where you made changes e.g. `gitref=george/s3-update` * **run-tests** - Defaults to true. Should always run the tests as part of the publish flow so that if tests fail, the connector is not published. * **comment-id** - This is automatically filled if you run /publish from a comment and enables the workflow to write back success/fail logs to the git comment. * **auto-bump-version** - Defaults to true, automates the post-publish process of bumping the connector's version in the yaml seed definitions and generating spec. +* **parallel** - Defaults to false. If set to true, a pool of runner agents will be spun up to allow publishing multiple connectors in parallel. Only switch this to true if publishing multiple connectors at once to avoid wasting $$$. ## Using credentials in CI From 57b31ee13d2571c6878d8bb82e9f3c91225e4d05 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 20 Jun 2022 18:56:58 +0000 Subject: [PATCH 73/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index eb1a0cb131f8..12d880f6f0a2 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -238,7 +238,7 @@ - name: Rockset destinationDefinitionId: 2c9d93a7-9a17-4789-9de9-f46f0097eb70 dockerRepository: airbyte/destination-rockset - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/rockset releaseStage: alpha - name: S3 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index fda39c9d11e4..353a4b9d7256 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -3780,7 +3780,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-rockset:0.1.2" +- dockerImage: "airbyte/destination-rockset:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/rockset" connectionSpecification: From 4dec177d1adc2334143d82ebf7cdcf31f8778b35 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 20 Jun 2022 19:49:48 +0000 Subject: [PATCH 74/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0d4e758d320d..b535cd270147 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -929,7 +929,7 @@ - name: TiDB sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae dockerRepository: airbyte/source-tidb - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/tidb icon: tidb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9a219813db7b..1868c8d3e080 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -8884,7 +8884,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tidb:0.1.1" +- dockerImage: "airbyte/source-tidb:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tidb" connectionSpecification: From cf5f8eb0cef39ae454f549b1d10b5e5f4bb51d50 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Mon, 20 Jun 2022 22:00:11 +0000 Subject: [PATCH 75/84] auto-bump connector version --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index eb1a0cb131f8..71560a4adc4d 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -183,7 +183,7 @@ - name: MySQL destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 dockerRepository: airbyte/destination-mysql - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.20 documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql icon: mysql.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index fda39c9d11e4..62bbd444ad94 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2807,7 +2807,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mysql:0.1.18" +- dockerImage: "airbyte/destination-mysql:0.1.20" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql" connectionSpecification: From 3f73ba2bbc02d1ba61d8bd4204748f97f2db5dbb Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 10:43:26 +0100 Subject: [PATCH 76/84] unbump failed publish versions --- .../connectors/destination-elasticsearch/Dockerfile | 2 +- .../connectors/destination-local-json/Dockerfile | 2 +- .../connectors/destination-mongodb-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mongodb/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mqtt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-mssql/Dockerfile | 2 +- .../connectors/destination-mysql-strict-encrypt/Dockerfile | 2 +- .../connectors/destination-oracle-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-oracle/Dockerfile | 2 +- airbyte-integrations/connectors/destination-postgres/Dockerfile | 2 +- .../connectors/source-clickhouse-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-clickhouse/Dockerfile | 2 +- airbyte-integrations/connectors/source-cockroachdb/Dockerfile | 2 +- .../connectors/source-mongodb-strict-encrypt/Dockerfile | 2 +- .../connectors/source-mysql-strict-encrypt/Dockerfile | 2 +- .../connectors/source-oracle-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/source-oracle/Dockerfile | 2 +- airbyte-integrations/connectors/source-redshift/Dockerfile | 2 +- airbyte-integrations/connectors/source-relational-db/Dockerfile | 2 +- airbyte-integrations/connectors/source-snowflake/Dockerfile | 2 +- 20 files changed, 20 insertions(+), 20 deletions(-) diff --git a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile index 6f8d3f8d7ba9..1fe4f9d4a20d 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-elasticsearch COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-elasticsearch diff --git a/airbyte-integrations/connectors/destination-local-json/Dockerfile b/airbyte-integrations/connectors/destination-local-json/Dockerfile index 947616e66c03..9ceea39cecdb 100644 --- a/airbyte-integrations/connectors/destination-local-json/Dockerfile +++ b/airbyte-integrations/connectors/destination-local-json/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-local-json COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.2.11 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/destination-local-json diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile index 74dab8dba94f..450beba6370f 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mongodb/Dockerfile b/airbyte-integrations/connectors/destination-mongodb/Dockerfile index 774a7d620a99..4567fdca3a98 100644 --- a/airbyte-integrations/connectors/destination-mongodb/Dockerfile +++ b/airbyte-integrations/connectors/destination-mongodb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mongodb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-mongodb diff --git a/airbyte-integrations/connectors/destination-mqtt/Dockerfile b/airbyte-integrations/connectors/destination-mqtt/Dockerfile index 792c70786846..615836c81792 100644 --- a/airbyte-integrations/connectors/destination-mqtt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mqtt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mqtt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-mqtt diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile index aa07ad06dba5..c142424038bc 100644 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.20 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/destination-mssql diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile index d4503dcaddab..78ad2460dc16 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile index f03271e99861..2270faca1ca6 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 7e2971d95735..73e7f54753f7 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile index 308263a7b563..67d9436e2fac 100644 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ b/airbyte-integrations/connectors/destination-postgres/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-postgres COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.21 +LABEL io.airbyte.version=0.3.20 LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile index 09e31f5ecfab..8555fba96344 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile index 9f5553e81281..1f9b1623150c 100644 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-clickhouse/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-clickhouse COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index 311ad518b9dc..8ae1e6922049 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile index 0c130eae19b1..f7abe1691c68 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mongodb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index 27bf840a921f..f477b07ca9dd 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.5.12 +LABEL io.airbyte.version=0.5.11 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile index 030f9909887a..96726701554a 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile @@ -17,5 +17,5 @@ ENV TZ UTC COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.16 +LABEL io.airbyte.version=0.3.15 LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile index 38ec42a166de..cfc0c7aeaa39 100644 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ b/airbyte-integrations/connectors/source-oracle/Dockerfile @@ -8,5 +8,5 @@ ENV TZ UTC COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.16 +LABEL io.airbyte.version=0.3.15 LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile index 9a9c16a8ed54..a743455d2227 100644 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ b/airbyte-integrations/connectors/source-redshift/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-redshift COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.11 +LABEL io.airbyte.version=0.3.10 LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-relational-db/Dockerfile b/airbyte-integrations/connectors/source-relational-db/Dockerfile index cba4be187f39..36baea09eafa 100644 --- a/airbyte-integrations/connectors/source-relational-db/Dockerfile +++ b/airbyte-integrations/connectors/source-relational-db/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-relational-db COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-relational-db diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile index 7a917c353705..7d904bc10687 100644 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/source-snowflake/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-snowflake COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-snowflake From 610b05aec01138f5bc8dd58c75b3f49d52dc8568 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 10:49:08 +0100 Subject: [PATCH 77/84] missed dockerfiles --- .../connectors/destination-mysql-strict-encrypt/Dockerfile | 2 +- airbyte-integrations/connectors/destination-oracle/Dockerfile | 2 +- .../connectors/source-mysql-strict-encrypt/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile index 78ad2460dc16..ad65ba60ef06 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile index 73e7f54753f7..8ae09c1d1e30 100644 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-oracle COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index f477b07ca9dd..4a95c3c9cd04 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.5.11 +LABEL io.airbyte.version=0.5.10 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt From cc849be58e7d7c7069b32c6e3548a612c3ab8f17 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 10:54:20 +0100 Subject: [PATCH 78/84] remove failed docs --- docs/integrations/destinations/elasticsearch.md | 1 - docs/integrations/destinations/mongodb.md | 1 - docs/integrations/destinations/mqtt.md | 1 - docs/integrations/destinations/mssql.md | 1 - docs/integrations/destinations/mysql.md | 1 - docs/integrations/destinations/oracle.md | 2 -- docs/integrations/destinations/postgres.md | 1 - docs/integrations/sources/clickhouse.md | 2 -- docs/integrations/sources/cockroachdb.md | 1 - docs/integrations/sources/oracle.md | 1 - docs/integrations/sources/redshift.md | 1 - docs/integrations/sources/snowflake.md | 1 - 12 files changed, 14 deletions(-) diff --git a/docs/integrations/destinations/elasticsearch.md b/docs/integrations/destinations/elasticsearch.md index b74fbe70441d..c2dfc2982b6a 100644 --- a/docs/integrations/destinations/elasticsearch.md +++ b/docs/integrations/destinations/elasticsearch.md @@ -69,7 +69,6 @@ Enter the hostname and/or other configuration information ... | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.2 | 2022-04-19 | [11752](https://github.com/airbytehq/airbyte/pull/11752) | Reduce batch size to 32Mb | | 0.1.1 | 2022-02-10 | [10256](https://github.com/airbytehq/airbyte/pull/1256) | Add ExitOnOutOfMemoryError connectors | | 0.1.0 | 2021-10-13 | [7005](https://github.com/airbytehq/airbyte/pull/7005) | Initial release. | diff --git a/docs/integrations/destinations/mongodb.md b/docs/integrations/destinations/mongodb.md index 297a45799ed6..42733f2b1a8a 100644 --- a/docs/integrations/destinations/mongodb.md +++ b/docs/integrations/destinations/mongodb.md @@ -93,7 +93,6 @@ Collection names should begin with an underscore or a letter character, and cann | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.5 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.4 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2021-12-30 | [8809](https://github.com/airbytehq/airbyte/pull/8809) | Update connector fields title/description | | 0.1.2 | 2021-10-18 | [6945](https://github.com/airbytehq/airbyte/pull/6945) | Create a secure-only MongoDb destination | diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md index a6fd3140e772..b1e638243be0 100644 --- a/docs/integrations/destinations/mqtt.md +++ b/docs/integrations/destinations/mqtt.md @@ -82,5 +82,4 @@ _NOTE_: MQTT version 5 is not supported yet. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index 67481b38c77e..f029e4762175 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -139,7 +139,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | -| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.8 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index a4813991e5fb..671f957b4888 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -134,7 +134,6 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------| :--- | -| 0.1.6 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.5 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 96794b27e217..532d0fd3ea5f 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -92,7 +92,6 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.1.18 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.17 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.16 | 2022-04-06 | [11514](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.15 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | @@ -113,7 +112,6 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------|:-----------|:--------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------| -| 0.1.6 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.5 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.4 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling and remove DBT support | | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | (unpublished) Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index 6e40332ec4fa..97ad372c88c0 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -128,7 +128,6 @@ Now that you have set up the Postgres destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| -| 0.3.21 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.20 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.3.19 | 2022-04-25 | [12195](https://github.com/airbytehq/airbyte/pull/12195) | Add support for additional JDBC URL Params input | | 0.3.18 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index b73a8e1e4257..38e2e2295131 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -78,7 +78,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:-----------------------------------------------------------------| -| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.10 | 2022-04-12 | [11729](https://github.com/airbytehq/airbyte/pull/11514) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.9 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | | 0.1.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | @@ -94,7 +93,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| -| 0.1.7 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | | 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.3 | 2021-12-29 | [\#9182](https://github.com/airbytehq/airbyte/pull/9182) [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY. Fixed tests | diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index df7d0f1bb923..7bf2b0a53094 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,7 +95,6 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- | :--- | -| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 83066723baca..031e55def8e2 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -132,7 +132,6 @@ Airbite has the ability to connect to the Oracle source with 3 network connectiv | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:------------------------------------------------| -| 0.3.16 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.15 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.3.14 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | | 0.3.13 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index 6d06cc483574..33bd04918bc2 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -54,7 +54,6 @@ All Redshift connections are encrypted using SSL | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.3.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.3.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption |0 | 0.3.9 | 2022-02-21 | [9744](https://github.com/airbytehq/airbyte/pull/9744) | List only the tables on which the user has SELECT permissions. | 0.3.8 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 476163608652..ddeca9ac1d03 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -103,7 +103,6 @@ Field | Description | | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-27 | [10953](https://github.com/airbytehq/airbyte/pull/10953) | Implement OAuth flow | | 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | From f7e1c521afae6a9a5b57241677e20eaefb66ad0d Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 10:58:07 +0100 Subject: [PATCH 79/84] mssql fix --- docs/integrations/destinations/mssql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index f029e4762175..4c23d97ec648 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -115,7 +115,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- |:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.19 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.18 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.17 | 2022-04-05 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | @@ -139,6 +138,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.8 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | From 113dbe065e1a9631b56b25fb229073667d32699c Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 14:14:07 +0100 Subject: [PATCH 80/84] overhauled the git comment output --- .github/workflows/publish-command.yml | 93 +++++++++++++++++---------- 1 file changed, 59 insertions(+), 34 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 533a9e8c8cbf..ac3723630392 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -176,11 +176,39 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" + - name: Print start message on github comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: Publishing the following connectors:
${{ steps.preprocess.outputs.connectorjson }}
Running tests before publishing: **${{ github.event.inputs.run-tests }}**
https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + create-success-and-fails-comment-table: + name: Set up table in git comment + if: github.event.inputs.comment-id + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Create table header + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | +
+ + | Connector | Published | Definitions generated | + - name: Create table separator + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + | --- | --- | --- | publish-image: timeout-minutes: 240 needs: - start-publish-image-runner-0 - preprocess-matrix + - create-success-and-fails-comment-table strategy: max-parallel: 5 fail-fast: false @@ -189,13 +217,6 @@ jobs: runs-on: runner-pool-${{ github.run_id }} environment: more-secrets steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -228,6 +249,7 @@ jobs: - name: Install Pyenv and Tox run: | python3 -m pip install --quiet virtualenv==16.7.9 --user + rm -r venv || echo "no pre-existing venv" python3 -m virtualenv venv source venv/bin/activate pip install --quiet tox==3.24.4 @@ -287,20 +309,6 @@ jobs: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{matrix.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{matrix.connector}} - name: Check if connector in definitions yaml if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -338,27 +346,43 @@ jobs: git commit -m "auto-bump connector version" git pull origin ${{ github.event.inputs.gitref }} git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{matrix.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + id: auto-bump + - name: Process outcomes into emojis + if: ${{ always() && github.event.inputs.comment-id }} + run: | + if [[ ${{ steps.publish.outcome }} = "success" ]]; then + echo "PUBLISH_OUTCOME=:white_check_mark:" >> $GITHUB_ENV + else + echo "PUBLISH_OUTCOME=:x:" >> $GITHUB_ENV + fi + if [[ ${{ steps.auto-bump.outcome }} = "success" ]]; then + echo "AUTO_BUMP_OUTCOME=:white_check_mark:" >> $GITHUB_ENV + else + echo "AUTO_BUMP_OUTCOME=:x:" >> $GITHUB_ENV + fi + - name: Add connector outcome line to table + if: ${{ always() && github.event.inputs.comment-id }} uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Couldn't auto-bump version for ${{matrix.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() + | ${{ matrix.connector }} | ${{ env.PUBLISH_OUTCOME }} | ${{ env.AUTO_BUMP_OUTCOME }} | + add-helpful-info-to-git-comment: + if: ${{ always() && github.event.inputs.comment-id }} + name: Add extra info to git comment + needs: + - start-publish-image-runner-0 # required to get output from the start-runner job + - publish-image # required to wait when the main job is done + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Add hint for manual seed definition update uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +
+ + if you have connectors that successfully published but failed definition generation, follow [step 4 here ▶️](https://docs.airbyte.com/connector-development/#publishing-a-connector) # In case of self-hosted EC2 errors, remove this block. stop-publish-image-runner-0: if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs @@ -368,6 +392,7 @@ jobs: - preprocess-matrix - publish-image # required to wait when the main job is done - find_valid_pat + - add-helpful-info-to-git-comment runs-on: ubuntu-latest steps: - name: Configure AWS credentials From 1c30f210cac42bf3a1ef6c9f0271552680204381 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 14:17:45 +0100 Subject: [PATCH 81/84] bumping a test connector that should work --- airbyte-integrations/connectors/source-openweather/Dockerfile | 2 +- docs/integrations/sources/openweather.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-openweather/Dockerfile b/airbyte-integrations/connectors/source-openweather/Dockerfile index b344b066bd47..264f36fd53b7 100644 --- a/airbyte-integrations/connectors/source-openweather/Dockerfile +++ b/airbyte-integrations/connectors/source-openweather/Dockerfile @@ -34,5 +34,5 @@ COPY source_openweather ./source_openweather ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-openweather diff --git a/docs/integrations/sources/openweather.md b/docs/integrations/sources/openweather.md index 8609d916597b..05e4e2ae9e2a 100644 --- a/docs/integrations/sources/openweather.md +++ b/docs/integrations/sources/openweather.md @@ -34,6 +34,7 @@ The free plan allows 60 calls per minute and 1,000,000 calls per month, you won' | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-21 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | No changes. Used connector to test publish workflow changes. | | 0.1.4 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | | 0.1.0 | 2021-10-27 | [7434](https://github.com/airbytehq/airbyte/pull/7434) | Initial release | From a4688d9ad573bdde6a508a623a799558684cb699 Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 14:32:27 +0100 Subject: [PATCH 82/84] slight order switcheroo --- .github/workflows/publish-command.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index ac3723630392..fc3c945c7eb3 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -176,19 +176,19 @@ jobs: middle=${stripped_connector//,/$replace} full="$start$middle$end" echo "::set-output name=connectorjson::$full" - - name: Print start message on github comment + write-initial-output-to-comment: + name: Set up git comment + if: github.event.inputs.comment-id + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Print start message if: github.event.inputs.comment-id && success() uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | > :clock2: Publishing the following connectors:
${{ steps.preprocess.outputs.connectorjson }}
Running tests before publishing: **${{ github.event.inputs.run-tests }}**
https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - create-success-and-fails-comment-table: - name: Set up table in git comment - if: github.event.inputs.comment-id - needs: start-publish-image-runner-0 - runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} - steps: - name: Create table header uses: peter-evans/create-or-update-comment@v1 with: @@ -208,7 +208,7 @@ jobs: needs: - start-publish-image-runner-0 - preprocess-matrix - - create-success-and-fails-comment-table + - write-initial-output-to-comment strategy: max-parallel: 5 fail-fast: false From e68f4916c627609c8d1660aa9cc919dad4f8192c Mon Sep 17 00:00:00 2001 From: George Claireaux Date: Tue, 21 Jun 2022 14:43:37 +0100 Subject: [PATCH 83/84] output connectors properly in first message --- .github/workflows/publish-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index fc3c945c7eb3..a0814d20cef5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -188,7 +188,7 @@ jobs: with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :clock2: Publishing the following connectors:
${{ steps.preprocess.outputs.connectorjson }}
Running tests before publishing: **${{ github.event.inputs.run-tests }}**
https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :clock2: Publishing the following connectors:
${{ github.event.inputs.connector }}
Running tests before publishing: **${{ github.event.inputs.run-tests }}**
https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} - name: Create table header uses: peter-evans/create-or-update-comment@v1 with: From 035e7acd5b2a6deff4f011bddb86352349ca1d3e Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 21 Jun 2022 13:50:48 +0000 Subject: [PATCH 84/84] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0d11ae262cc7..26db7e596f58 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -613,7 +613,7 @@ - name: OpenWeather sourceDefinitionId: d8540a80-6120-485d-b7d6-272bca477d9b dockerRepository: airbyte/source-openweather - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/openweather sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 753834977adf..d5b9183689b4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5910,7 +5910,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-openweather:0.1.4" +- dockerImage: "airbyte/source-openweather:0.1.5" spec: documentationUrl: "https://docsurl.com" connectionSpecification: