Enhance /publish to allow for multiple connectors and parallel execution

.github/workflows/publish-command.yml

@@ -65,9 +65,29 @@ jobs:
           aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
+  preprocess-matrix:
+    needs: start-publish-image-runner
+    runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
+    outputs:
+      connectorjson: ${{ steps.preprocess.outputs.connectorjson }}
+    steps:
+      - id: preprocess
+        run: |
+          start="[\""
+          replace="\",\""
+          end="\"]"
+          stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')"
+          middle=${stripped_connector//,/$replace}
+          full="$start$middle$end"
+          echo "::set-output name=connectorjson::$full"
   publish-image:
     timeout-minutes: 240
-    needs: start-publish-image-runner
+    needs:
+      - start-publish-image-runner
+      - preprocess-matrix
+    strategy:
+      matrix:
+        connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }}
     runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
     environment: more-secrets
     steps:
@@ -77,7 +97,7 @@ jobs:
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
-            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+            > :clock2: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v0
         with:
@@ -89,9 +109,9 @@ jobs:
         with:
           regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
           regex_flags: "i" # required to be set for this plugin
-          search_string: ${{ github.event.inputs.connector }}
+          search_string: ${{ matrix.connector }}
       - name: Validate input workflow format
-        if: steps.regex.outputs.first_match != github.event.inputs.connector
+        if: steps.regex.outputs.first_match != matrix.connector
         run: echo "The connector provided has an invalid format!" && exit 1
       - name: Checkout Airbyte
         uses: actions/checkout@v2
@@ -126,43 +146,43 @@ jobs:
           source venv/bin/activate
           tox -r -c ./tools/tox_ci.ini
           pip install --quiet -e ./tools/ci_*
-      - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
+      - name: Write Integration Test Credentials for ${{ matrix.connector }}
         run: |
           source venv/bin/activate
-          ci_credentials ${{ github.event.inputs.connector }}
+          ci_credentials ${{ matrix.connector }}
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       - name: Set Name and Version Environment Vars
-        if: startsWith(github.event.inputs.connector, 'connectors')
+        if: startsWith(matrix.connector, 'connectors')
         run: |
           source tools/lib/lib.sh
-          DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile
-          echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
+          DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile
+          echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
           echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV
       - name: Prepare Sentry
-        if: startsWith(github.event.inputs.connector, 'connectors')
+        if: startsWith(matrix.connector, 'connectors')
         run: |
           curl -sL https://sentry.io/get-cli/ | bash
       - name: Create Sentry Release
-        if: startsWith(github.event.inputs.connector, 'connectors')
+        if: startsWith(matrix.connector, 'connectors')
         run: |
           sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
         env:
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
           SENTRY_ORG: airbyte-5j
           SENTRY_PROJECT: airbyte-connectors
-      - name: Publish ${{ github.event.inputs.connector }}
+      - name: Publish ${{ matrix.connector }}
         run: |
           echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
-          ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
+          ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
         id: publish
         env:
           DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
           DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
           # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
           TZ: UTC
       - name: Finalize Sentry release
-        if: startsWith(github.event.inputs.connector, 'connectors')
+        if: startsWith(matrix.connector, 'connectors')
         run: |
           sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
         env:
@@ -175,14 +195,14 @@ jobs:
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
-            > :rocket: Successfully published ${{github.event.inputs.connector}}
+            > :rocket: Successfully published ${{matrix.connector}}
       - name: Add Published Failure Comment
         if: github.event.inputs.comment-id && !success()
         uses: peter-evans/create-or-update-comment@v1
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
-            > :x: Failed to publish ${{github.event.inputs.connector}}
+            > :x: Failed to publish ${{matrix.connector}}
       - name: Check if connector in definitions yaml
         if: github.event.inputs.auto-bump-version == 'true' && success()
         run: |
@@ -226,21 +246,21 @@ jobs:
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
-            > :rocket: Auto-bumped version for ${{github.event.inputs.connector}}
+            > :rocket: Auto-bumped version for ${{matrix.connector}}
       - name: Add Version Bump Failure Comment
         if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success()
         uses: peter-evans/create-or-update-comment@v1
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
-            > :x: Couldn't auto-bump version for ${{github.event.inputs.connector}}
+            > :x: Couldn't auto-bump version for ${{matrix.connector}}
       - name: Add Final Success Comment
         if: github.event.inputs.comment-id && success()
         uses: peter-evans/create-or-update-comment@v1
         with:
           comment-id: ${{ github.event.inputs.comment-id }}
           body: |
-            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+            > :white_check_mark: ${{matrix.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
   # In case of self-hosted EC2 errors, remove this block.
   stop-publish-image-runner:
     name: Stop Build EC2 Runner

airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile

@@ -16,5 +16,5 @@ ENV APPLICATION destination-azure-blob-storage
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.5
 LABEL io.airbyte.name=airbyte/destination-azure-blob-storage

airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile

@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.1.8
+LABEL io.airbyte.version=1.1.9
 LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized

airbyte-integrations/connectors/destination-bigquery/Dockerfile

@@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=1.1.8
+LABEL io.airbyte.version=1.1.9
 LABEL io.airbyte.name=airbyte/destination-bigquery

docs/integrations/destinations/azureblobstorage.md

@@ -137,6 +137,7 @@ They will be like this in the output file:
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.5 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors |
 | 0.1.4 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance |
 | 0.1.3 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |
 | 0.1.2 | 2022-01-20 | [\#9682](https://github.com/airbytehq/airbyte/pull/9682) | Each data synchronization for each stream is written to a new blob to the folder with stream name. |

docs/integrations/destinations/bigquery.md

@@ -133,6 +133,7 @@ Now that you have set up the BigQuery destination connector, check out the follo
 
 | Version | Date       | Pull Request                                               | Subject                                                                                         |
 |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------|
+| 1.1.9 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors |
 | 1.1.8   | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579)   | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. |
 | 1.1.7   | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424)   | Reordered fields for specification.                                                             |
 | 1.1.6   | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768)   | Clarify that the service account key json field is required on cloud. |
@@ -171,6 +172,7 @@ Now that you have set up the BigQuery destination connector, check out the follo
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                 |
 |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------|
+| 1.1.9 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors |
 | 1.1.8   | 2022-06-07 | [13579](https://github.com/airbytehq/airbyte/pull/13579)   | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. |
 | 1.1.7   | 2022-06-07 | [13424](https://github.com/airbytehq/airbyte/pull/13424)   | Reordered fields for specification.                                                                                     |
 | 1.1.6   | 2022-05-15 | [12768](https://github.com/airbytehq/airbyte/pull/12768)   | Clarify that the service account key json field is required on cloud.                                                   |