diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 7f268783fc60..c3b94df610b6 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -41,7 +41,7 @@ runs: aws-region: us-east-2 - name: Start EC2 runner id: start-ec2-runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: start github-token: ${{ inputs.github-token }} @@ -49,6 +49,9 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} + # this adds a label to group any EC2 runners spun up within the same action run + # this enables creating a pool of runners to run multiple/matrix jobs on in parallel + label: runner-pool-${{ github.run_id }} aws-resource-tags: > [ {"Key": "BuildType", "Value": "oss"}, diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 60bf4453586a..a0814d20cef5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -24,6 +24,10 @@ on: description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" required: true default: "true" + parallel: + description: "Switching this to true will spin up 5 build agents instead of 1 and allow multi connector publishes to run in parallel" + required: true + default: "false" jobs: find_valid_pat: @@ -45,8 +49,8 @@ jobs: ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner: - name: Start Build EC2 Runner + start-publish-image-runner-0: + name: Start Build EC2 Runner 0 runs-on: ubuntu-latest needs: find_valid_pat outputs: @@ -65,19 +69,154 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - publish-image: - timeout-minutes: 240 - needs: start-publish-image-runner - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} - environment: more-secrets + label: ${{ github.run_id }}-publisher + start-publish-image-runner-1: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 1 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + start-publish-image-runner-2: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 2 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + start-publish-image-runner-3: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 3 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + start-publish-image-runner-4: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 4 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + preprocess-matrix: + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + outputs: + connectorjson: ${{ steps.preprocess.outputs.connectorjson }} + steps: + # given a string input of a single connector or comma separated list of connectors e.g. connector1, connector2 + # this step builds an array, by removing whitespace, add in quotation marks around connectors and braces [ ] at the start and end + # finally, it sets it as output from this job so we can use this array of connectors as our matrix strategy for publishing + - id: preprocess + run: | + start="[\"" + replace="\",\"" + end="\"]" + stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')" + middle=${stripped_connector//,/$replace} + full="$start$middle$end" + echo "::set-output name=connectorjson::$full" + write-initial-output-to-comment: + name: Set up git comment + if: github.event.inputs.comment-id + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Print start message + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: Publishing the following connectors:
${{ github.event.inputs.connector }}
Running tests before publishing: **${{ github.event.inputs.run-tests }}**
https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Create table header uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +
+ + | Connector | Published | Definitions generated | + - name: Create table separator + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + | --- | --- | --- | + publish-image: + timeout-minutes: 240 + needs: + - start-publish-image-runner-0 + - preprocess-matrix + - write-initial-output-to-comment + strategy: + max-parallel: 5 + fail-fast: false + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + runs-on: runner-pool-${{ github.run_id }} + environment: more-secrets + steps: - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -89,9 +228,9 @@ jobs: with: regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} + search_string: ${{ matrix.connector }} - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector + if: steps.regex.outputs.first_match != matrix.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v2 @@ -110,6 +249,7 @@ jobs: - name: Install Pyenv and Tox run: | python3 -m pip install --quiet virtualenv==16.7.9 --user + rm -r venv || echo "no pre-existing venv" python3 -m virtualenv venv source venv/bin/activate pip install --quiet tox==3.24.4 @@ -126,35 +266,35 @@ jobs: source venv/bin/activate tox -r -c ./tools/tox_ci.ini pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + - name: Write Integration Test Credentials for ${{ matrix.connector }} run: | source venv/bin/activate - ci_credentials ${{ github.event.inputs.connector }} + ci_credentials ${{ matrix.connector }} env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - name: Prepare Sentry - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | - curl -sL https://sentry.io/get-cli/ | bash + curl -sL https://sentry.io/get-cli/ | bash || echo "sentry cli already installed" - name: Create Sentry Release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ github.event.inputs.connector }} + - name: Publish ${{ matrix.connector }} run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache id: publish env: DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} @@ -162,27 +302,13 @@ jobs: # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. TZ: UTC - name: Finalize Sentry release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{github.event.inputs.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{github.event.inputs.connector}} - name: Check if connector in definitions yaml if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -220,36 +346,103 @@ jobs: git commit -m "auto-bump connector version" git pull origin ${{ github.event.inputs.gitref }} git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{github.event.inputs.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + id: auto-bump + - name: Process outcomes into emojis + if: ${{ always() && github.event.inputs.comment-id }} + run: | + if [[ ${{ steps.publish.outcome }} = "success" ]]; then + echo "PUBLISH_OUTCOME=:white_check_mark:" >> $GITHUB_ENV + else + echo "PUBLISH_OUTCOME=:x:" >> $GITHUB_ENV + fi + if [[ ${{ steps.auto-bump.outcome }} = "success" ]]; then + echo "AUTO_BUMP_OUTCOME=:white_check_mark:" >> $GITHUB_ENV + else + echo "AUTO_BUMP_OUTCOME=:x:" >> $GITHUB_ENV + fi + - name: Add connector outcome line to table + if: ${{ always() && github.event.inputs.comment-id }} uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Couldn't auto-bump version for ${{github.event.inputs.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() + | ${{ matrix.connector }} | ${{ env.PUBLISH_OUTCOME }} | ${{ env.AUTO_BUMP_OUTCOME }} | + add-helpful-info-to-git-comment: + if: ${{ always() && github.event.inputs.comment-id }} + name: Add extra info to git comment + needs: + - start-publish-image-runner-0 # required to get output from the start-runner job + - publish-image # required to wait when the main job is done + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Add hint for manual seed definition update uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +
+ + if you have connectors that successfully published but failed definition generation, follow [step 4 here ▶️](https://docs.airbyte.com/connector-development/#publishing-a-connector) # In case of self-hosted EC2 errors, remove this block. - stop-publish-image-runner: + stop-publish-image-runner-0: + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs name: Stop Build EC2 Runner needs: - - start-publish-image-runner # required to get output from the start-runner job + - start-publish-image-runner-0 # required to get output from the start-runner job + - preprocess-matrix - publish-image # required to wait when the main job is done - find_valid_pat + - add-helpful-info-to-git-comment + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: airbytehq/ec2-github-runner@base64v1.1.0 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-publish-image-runner-0.outputs.label }} + ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} + stop-publish-image-runner-multi: + if: ${{ always() && github.event.inputs.parallel == 'true' }} + name: Stop Build EC2 Runner + needs: + - start-publish-image-runner-0 + - start-publish-image-runner-1 + - start-publish-image-runner-2 + - start-publish-image-runner-3 + - start-publish-image-runner-4 + - preprocess-matrix + - publish-image # required to wait when the main job is done + - find_valid_pat + strategy: + fail-fast: false + matrix: + ec2-instance: + [ + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-2.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-3.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-4.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", + }, + ] runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -258,9 +451,9 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }} + label: ${{ matrix.ec2-instance.label }} + ec2-instance-id: ${{ matrix.ec2-instance.id }} diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index eb1a0cb131f8..6b0047f07c18 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -183,7 +183,7 @@ - name: MySQL destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 dockerRepository: airbyte/destination-mysql - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.20 documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql icon: mysql.svg releaseStage: alpha @@ -238,7 +238,7 @@ - name: Rockset destinationDefinitionId: 2c9d93a7-9a17-4789-9de9-f46f0097eb70 dockerRepository: airbyte/destination-rockset - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/rockset releaseStage: alpha - name: S3 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index fda39c9d11e4..dcd20018abfd 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2807,7 +2807,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mysql:0.1.18" +- dockerImage: "airbyte/destination-mysql:0.1.20" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql" connectionSpecification: @@ -3780,7 +3780,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-rockset:0.1.2" +- dockerImage: "airbyte/destination-rockset:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/rockset" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 24eb98ed36dc..26db7e596f58 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -613,7 +613,7 @@ - name: OpenWeather sourceDefinitionId: d8540a80-6120-485d-b7d6-272bca477d9b dockerRepository: airbyte/source-openweather - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/openweather sourceType: api releaseStage: alpha @@ -929,7 +929,7 @@ - name: TiDB sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae dockerRepository: airbyte/source-tidb - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/tidb icon: tidb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 27153faffcf3..d5b9183689b4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5910,7 +5910,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-openweather:0.1.4" +- dockerImage: "airbyte/source-openweather:0.1.5" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -8884,7 +8884,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tidb:0.1.1" +- dockerImage: "airbyte/source-tidb:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tidb" connectionSpecification: diff --git a/airbyte-integrations/connectors/destination-jdbc/Dockerfile b/airbyte-integrations/connectors/destination-jdbc/Dockerfile index aa9e1177a2b1..a35e7fb7b3f2 100644 --- a/airbyte-integrations/connectors/destination-jdbc/Dockerfile +++ b/airbyte-integrations/connectors/destination-jdbc/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-jdbc COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.13 +LABEL io.airbyte.version=0.3.14 LABEL io.airbyte.name=airbyte/destination-jdbc diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index 2115990996e8..49dcaadfa742 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -111,26 +111,6 @@ protected void tearDown(final TestDestinationEnv testEnv) { container.close(); } - @Override - protected TestDataComparator getTestDataComparator() { - return new AdvancedTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - /* Helpers */ private JsonNode getAuthTypeConfig() { diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index bc324b2bff11..29fa71d00ceb 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.20 LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-rockset/Dockerfile b/airbyte-integrations/connectors/destination-rockset/Dockerfile index 73477dc97bb6..136dbcd02b48 100644 --- a/airbyte-integrations/connectors/destination-rockset/Dockerfile +++ b/airbyte-integrations/connectors/destination-rockset/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-rockset COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-rockset diff --git a/airbyte-integrations/connectors/source-openweather/Dockerfile b/airbyte-integrations/connectors/source-openweather/Dockerfile index b344b066bd47..264f36fd53b7 100644 --- a/airbyte-integrations/connectors/source-openweather/Dockerfile +++ b/airbyte-integrations/connectors/source-openweather/Dockerfile @@ -34,5 +34,5 @@ COPY source_openweather ./source_openweather ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-openweather diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index 6179f1f2b654..d322630a76e5 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index da91130ca49a..ca18ddc8f81e 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -136,12 +136,13 @@ Once you've finished iterating on the changes to a connector as specified in its ### The /publish command Publishing a connector can be done using the `/publish` command as outlined in the above section. The command runs a [github workflow](https://github.com/airbytehq/airbyte/actions/workflows/publish-command.yml), and has the following configurable parameters: -* **connector** - Required. This tells the workflow which connector to publish. e.g. `connector=connectors/source-amazon-ads` +* **connector** - Required. This tells the workflow which connector to publish. e.g. `connector=connectors/source-amazon-ads`. This can also be a comma-separated list of many connectors, e.g. `connector=connectors/source-s3,connectors/destination-postgres,connectors/source-facebook-marketing`. See the parallel flag below if publishing multiple connectors. * **repo** - Defaults to the main airbyte repo. Set this when building connectors from forked repos. e.g. `repo=userfork/airbyte` * **gitref** - Defaults to the branch of the PR where the /publish command is run as a comment. If running manually, set this to your branch where you made changes e.g. `gitref=george/s3-update` * **run-tests** - Defaults to true. Should always run the tests as part of the publish flow so that if tests fail, the connector is not published. * **comment-id** - This is automatically filled if you run /publish from a comment and enables the workflow to write back success/fail logs to the git comment. * **auto-bump-version** - Defaults to true, automates the post-publish process of bumping the connector's version in the yaml seed definitions and generating spec. +* **parallel** - Defaults to false. If set to true, a pool of runner agents will be spun up to allow publishing multiple connectors in parallel. Only switch this to true if publishing multiple connectors at once to avoid wasting $$$. ## Using credentials in CI diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md index 36e03c29a485..b1e638243be0 100644 --- a/docs/integrations/destinations/mqtt.md +++ b/docs/integrations/destinations/mqtt.md @@ -82,4 +82,4 @@ _NOTE_: MQTT version 5 is not supported yet. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.2 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | +| 0.1.1 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index a855693f3f74..4c23d97ec648 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -138,6 +138,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.8 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index fd2938f3626e..671f957b4888 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -110,6 +110,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.19 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.18 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | diff --git a/docs/integrations/destinations/rockset.md b/docs/integrations/destinations/rockset.md index 601bcd1be6ba..28f1ce653fc5 100644 --- a/docs/integrations/destinations/rockset.md +++ b/docs/integrations/destinations/rockset.md @@ -30,6 +30,7 @@ | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.2 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.1 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.0 | 2021-11-15 | [\#8006](https://github.com/airbytehq/airbyte/pull/8006) | Initial release| diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index 3de1b54f3b0f..7bf2b0a53094 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -111,6 +111,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.8 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | diff --git a/docs/integrations/sources/openweather.md b/docs/integrations/sources/openweather.md index 8609d916597b..05e4e2ae9e2a 100644 --- a/docs/integrations/sources/openweather.md +++ b/docs/integrations/sources/openweather.md @@ -34,6 +34,7 @@ The free plan allows 60 calls per minute and 1,000,000 calls per month, you won' | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-21 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | No changes. Used connector to test publish workflow changes. | | 0.1.4 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | | 0.1.0 | 2021-10-27 | [7434](https://github.com/airbytehq/airbyte/pull/7434) | Initial release | diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 5cea0d66f733..4b82a1728354 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -120,5 +120,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | | :------ | :--- | :----------- | ------- | +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release |