Merge branch 'master' into leti/add-segment-track-to-suggested-destin…

…ations

* master: (75 commits)
  source-sentry: migrate to per-stream states (#17466)
  Greg/clickhouse polishing (#17483)
  upgrade debezium version to 1.9.6 (#17459)
  🐛 Source Twilio: Lookback_window config (#17478)
  hide S3 source connector from catalog (#17472)
  🪟 Migrate styles for Connection-related Components (#17339)
  Added new title (#17480)
  Refactor & convert `PageTitle` to SCSS (#17139)
  updated releaseStage for zendesk-talk (#17477)
  [low-code] Apply log level to stream loggers (#17284)
  🐛 Source Salesforce: filter out objects not supported by the Bulk API (#17453)
  Source Marketo: certify GA (#17445)
  Update greenhouse paginator (#17429)
  Add some services start validation to acceptance_tests.sh (#17425)
  📖 Removes $ from terminal commands to allow direct copying. (#17467)
  migrate source GA connectors to per-stream states (2) (#17410)
  Source Klaviyo: bump CDK dependency (#17422)
  Source Pinterest: change releaseStage to GA (#17045)
  Source Pinterest: Set start_date dynamically based on API restrictions for lookup (#17387)
  updated releaseStage to generally_available (#17374)
  ...

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.9
+current_version = 0.40.10
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.9
+VERSION=0.40.10
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -57,7 +57,6 @@ WEBAPP_URL=http://localhost:8000/
 # Although not present as an env var, required for webapp configuration.
 API_URL=/api/v1/
 
-
 ### JOBS ###
 # Relevant to scaling.
 SYNC_JOB_MAX_ATTEMPTS=3
@@ -81,7 +80,7 @@ LOG_LEVEL=INFO
 
 ### APPLICATIONS ###
 # Worker #
-WORKERS_MICRONAUT_ENVIRONMENTS=control
+WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
 # Relevant to scaling.
 MAX_SYNC_WORKERS=5
 MAX_SPEC_WORKERS=5
@@ -95,7 +94,6 @@ WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=
 
 ### FEATURE FLAGS ###
 AUTO_DISABLE_FAILING_CONNECTIONS=false
-EXPOSE_SECRETS_IN_EXPORT=false
 FORCE_MIGRATE_SECRET_STORE=false
 
 ### MONITORING FLAGS ###

.env.dev

@@ -25,7 +25,7 @@ API_URL=/api/v1/
 INTERNAL_API_HOST=airbyte-server:8001
 SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
-WORKERS_MICRONAUT_ENVIRONMENTS=control
+WORKERS_MICRONAUT_ENVIRONMENTS=control-plane
 
 # Sentry
 SENTRY_DSN=""

.github/workflows/assign-issue-to-project.yaml

@@ -32,9 +32,3 @@ jobs:
         with:
           project: "https://github.com/airbytehq/airbyte/projects/15"
           column_name: "To do"
-      - name: Assign Onboarding Improvements PR to Project
-        uses: srggrs/[email protected]
-        if: contains(github.event.pull_request.labels.*.name, 'project/onboarding-improvements')
-        with:
-          project: "https://github.com/airbytehq/airbyte/projects/15"
-          column_name: "To do"

.github/workflows/label-github-issues-by-context.yml

@@ -0,0 +1,27 @@
+name: "Add labels to github issues based on context"
+on:
+  issues:
+    types: [opened, labeled, unlabeled]
+
+jobs:
+  shared-issues:
+    name: "Add Labels to Issues.  Safe to Merge on fail"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte Repo for PAT command
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        # Cannot share PAT outside of JOB context
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \
+            ${{ secrets.OCTAVIA_PAT }}
+      - name: Run Issue Command from workflow-actions
+        uses: nick-fields/private-action-loader@v3
+        with:
+          pal-repo-token: "${{ env.PAT }}"
+          pal-repo-name: airbytehq/workflow-actions@production
+          # the following input gets passed to the private
+          token: "${{ env.PAT }}"
+          # ref: https://github.com/airbytehq/workflow-actions/blob/main/src/bin_issue.ts
+          command: "issue"

.github/workflows/labeler.yml → .github/workflows/label-github-issues-by-path.yml

@@ -1,5 +1,5 @@
 # the mapping from filepath to label
-# is defined in .github/labeler.yml
+# is defined in .github/label-github-issues-by-path.yml
 
 name: "Label PR based on filepath"
 on:

.github/workflows/label-prs-by-context.yml

@@ -0,0 +1,28 @@
+# Runs internal automation for pull requests
+
+name: "Add labels to github PRs based on context"
+on:
+  pull_request_target:
+    types: [opened, labeled, unlabeled, ready_for_review, synchronize, reopened]
+
+jobs:
+  shared-pr-labeller:
+    name: "Add Labels to PRs.  Safe to Merge on fail"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Airbyte Repo for PAT command
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        # Cannot share PAT outside of JOB context
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \
+            ${{ secrets.OCTAVIA_PAT }}
+      - name: Run Issue Command from workflow-actions
+        uses: nick-fields/private-action-loader@v3
+        with:
+          pal-repo-token: "${{ env.PAT }}"
+          pal-repo-name: airbytehq/workflow-actions@production
+          # the following input gets passed to the private action
+          token: "${{ env.PAT }}"
+          command: "pull"

.github/workflows/notify-on-label.yml

@@ -1,19 +1,30 @@
 # Notify users/teams when labels are added to an issue.
 
-name: Notify when adding label to issue
+name: Notify FE team for FE label on issues
 
 on:
   issues:
       types: [labeled]
 
 jobs:
   notify:
+    name: "Notify FE team on label creation"
     runs-on: ubuntu-latest
     steps:
-        - uses: jenschelkopf/[email protected]
-          with:
-            token: ${{ secrets.OCTAVIA_PAT }}
-            message: 'cc {recipients}'
-            # Specify a map of label -> team/user to notify
-            recipients: |
-              team/frontend=@airbytehq/frontend
+      - name: Checkout Airbyte Repo for PAT command
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        # Cannot share PAT outside of JOB context
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \
+            ${{ secrets.OCTAVIA_PAT }}
+      # Updated name to reflect reality.  Update name if you change recipients
+      - name: Notify FE team when on FE label creation
+        uses: jenschelkopf/[email protected]
+        with:
+          token: "${{ env.PAT }}"
+          message: 'cc {recipients}'
+          # Specify a map of label -> team/user to notify
+          recipients: |
+            team/frontend=@airbytehq/frontend

.github/workflows/notify-on-push-to-master.yml

@@ -1,4 +1,4 @@
-name: Notify Cloud of OSS Push to Master
+name: Trigger action in cloud based on push
 on:
   push:
     branches:
@@ -7,12 +7,21 @@ on:
 
 jobs:
   repo-sync:
+    name: "Fire a Repo Dispatch event to airbyte-cloud"
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout Airbyte Repo for PAT command
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        # Cannot share PAT outside of JOB context
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} \
+            ${{ secrets.OCTAVIA_PAT }}
       - name: Repository Dispatch
         uses: peter-evans/repository-dispatch@v2
         with:
-          token: ${{ secrets.OCTAVIA_PAT }}
+          token: ${{ env.PAT }}
           repository: airbytehq/airbyte-cloud
           event-type: oss-push-to-master
           client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'

airbyte-api/src/main/openapi/config.yaml

@@ -2756,6 +2756,8 @@ components:
           type: string
         sourceName:
           type: string
+        icon:
+          type: string
     SourceReadList:
       type: object
       required:
@@ -3053,6 +3055,8 @@ components:
           type: string
         destinationName:
           type: string
+        icon:
+          type: string
     DestinationReadList:
       type: object
       required:
@@ -3813,6 +3817,10 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/AttemptRead"
+    JobCreatedAt:
+      description: epoch time of the latest sync job. null if no sync job has taken place.
+      type: integer
+      format: int64
     JobStatus:
       type: string
       enum:
@@ -4508,6 +4516,43 @@ components:
           type: object
           additionalProperties: true
     # Web Backend
+    WebBackendConnectionListItem:
+      type: object
+      description: Information about a connection that shows up in the connection list view.
+      required:
+        - connectionId
+        - name
+        - sourceId
+        - destinationId
+        - source
+        - destination
+        - status
+        - isSyncing
+      properties:
+        connectionId:
+          $ref: "#/components/schemas/ConnectionId"
+        name:
+          type: string
+        sourceId:
+          $ref: "#/components/schemas/SourceId"
+        destinationId:
+          $ref: "#/components/schemas/DestinationId"
+        scheduleType:
+          $ref: "#/components/schemas/ConnectionScheduleType"
+        scheduleData:
+          $ref: "#/components/schemas/ConnectionScheduleData"
+        status:
+          $ref: "#/components/schemas/ConnectionStatus"
+        source:
+          $ref: "#/components/schemas/SourceRead"
+        destination:
+          $ref: "#/components/schemas/DestinationRead"
+        latestSyncJobCreatedAt:
+          $ref: "#/components/schemas/JobCreatedAt"
+        latestSyncJobStatus:
+          $ref: "#/components/schemas/JobStatus"
+        isSyncing:
+          type: boolean
     WebBackendConnectionRead:
       type: object
       required:
@@ -4562,9 +4607,7 @@ components:
           items:
             $ref: "#/components/schemas/OperationRead"
         latestSyncJobCreatedAt:
-          description: epoch time of the latest sync job. null if no sync job has taken place.
-          type: integer
-          format: int64
+          $ref: "#/components/schemas/JobCreatedAt"
         latestSyncJobStatus:
           $ref: "#/components/schemas/JobStatus"
         isSyncing:
@@ -4584,7 +4627,7 @@ components:
         connections:
           type: array
           items:
-            $ref: "#/components/schemas/WebBackendConnectionRead"
+            $ref: "#/components/schemas/WebBackendConnectionListItem"
     SyncMode:
       type: string
       enum:

airbyte-bootloader/Dockerfile

@@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.4
 ARG JDK_IMAGE=amazoncorretto:${JDK_VERSION}
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.40.9
+ARG VERSION=0.40.10
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-bootloader/src/main/java/io/airbyte/bootloader/BootloaderApp.java

@@ -9,6 +9,7 @@
 import io.airbyte.commons.lang.CloseableShutdownHook;
 import io.airbyte.commons.resources.MoreResources;
 import io.airbyte.commons.version.AirbyteVersion;
+import io.airbyte.commons.version.Version;
 import io.airbyte.config.Configs;
 import io.airbyte.config.EnvConfigs;
 import io.airbyte.config.StandardWorkspace;
@@ -153,6 +154,11 @@ public void load() throws Exception {
     final AirbyteVersion currAirbyteVersion = configs.getAirbyteVersion();
     assertNonBreakingMigration(jobPersistence, currAirbyteVersion);
 
+    final Version airbyteProtocolVersionMax = configs.getAirbyteProtocolVersionMax();
+    final Version airbyteProtocolVersionMin = configs.getAirbyteProtocolVersionMin();
+    // TODO ProtocolVersion validation should happen here
+    trackProtocolVersion(airbyteProtocolVersionMin, airbyteProtocolVersionMax);
+
     // TODO Will be converted to an injected singleton during DI migration
     final DatabaseMigrator configDbMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway);
     final DatabaseMigrator jobDbMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway);
@@ -185,7 +191,6 @@ private static Database getConfigDatabase(final DSLContext dslContext) throws IO
 
   private static ConfigPersistence getConfigPersistence(final Database configDatabase) throws IOException {
     final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder()
-        .maskSecrets(true)
         .copySecrets(true)
         .build();
 
@@ -299,6 +304,12 @@ private static void assertNonBreakingMigration(final JobPersistence jobPersisten
     }
   }
 
+  private void trackProtocolVersion(final Version airbyteProtocolVersionMin, final Version airbyteProtocolVersionMax) throws IOException {
+    jobPersistence.setAirbyteProtocolVersionMin(airbyteProtocolVersionMin);
+    jobPersistence.setAirbyteProtocolVersionMax(airbyteProtocolVersionMax);
+    LOGGER.info("AirbyteProtocol version support range [{}:{}]", airbyteProtocolVersionMin.serialize(), airbyteProtocolVersionMax.serialize());
+  }
+
   static boolean isLegalUpgrade(final AirbyteVersion airbyteDatabaseVersion, final AirbyteVersion airbyteVersion) {
     // means there was no previous version so upgrade even needs to happen. always legal.
     if (airbyteDatabaseVersion == null) {