Merge branch 'master' into vitalii/7531_google_workspace_admin_report…

…s_refactor_to_use_the_cdk

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.30.25-alpha
+current_version = 0.30.34-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -10,6 +10,12 @@ serialize =
 
 [bumpversion:file:.env]
 
+[bumpversion:file:airbyte-server/Dockerfile]
+
+[bumpversion:file:airbyte-workers/Dockerfile]
+
+[bumpversion:file:airbyte-scheduler/app/Dockerfile]
+
 [bumpversion:file:airbyte-webapp/package.json]
 
 [bumpversion:file:airbyte-webapp/package-lock.json]

.env

@@ -1,4 +1,4 @@
-VERSION=0.30.25-alpha
+VERSION=0.30.34-alpha
 
 # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker

.github/pull_request_template.md

@@ -29,7 +29,6 @@ Expand the relevant checklist and delete the others.
     - [ ] `docs/integrations/README.md`
     - [ ] `airbyte-integrations/builds.md`
 - [ ] PR name follows [PR naming conventions](https://docs.airbyte.io/contributing-to-airbyte/updating-documentation#issues-and-pull-requests)
-- [ ] Connector added to connector index like described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)
 
 #### Airbyter
 
@@ -40,6 +39,8 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
 - [ ] Credentials added to Github CI. [Instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci). 
 - [ ] [`/test connector=connectors/<name>` command](https://docs.airbyte.io/connector-development#updating-an-existing-connector) is passing. 
 - [ ] New Connector version released on Dockerhub by running the `/publish` command described [here](https://docs.airbyte.io/connector-development#updating-an-existing-connector)
+- [ ] After the connector is published, connector added to connector index as described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)
+- [ ] Seed specs have been re-generated by building the platform and committing the changes to the seed spec files, as described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)
 
 </p>
 </details>
@@ -59,7 +60,6 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
     - [ ] Connector's `bootstrap.md`. See [description and examples](https://docs.google.com/document/d/1ypdgmwmEHWv-TrO4_YOQ7pAJGVrMp5BOkEVh831N260/edit?usp=sharing)
     - [ ] Changelog updated in `docs/integrations/<source or destination>/<name>.md` including changelog. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog)
 - [ ] PR name follows [PR naming conventions](https://docs.airbyte.io/contributing-to-airbyte/updating-documentation#issues-and-pull-requests)
-- [ ] Connector version bumped like described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)
 
 #### Airbyter
 
@@ -70,6 +70,8 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
 - [ ] Credentials added to Github CI. [Instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci). 
 - [ ] [`/test connector=connectors/<name>` command](https://docs.airbyte.io/connector-development#updating-an-existing-connector) is passing. 
 - [ ] New Connector version released on Dockerhub by running the `/publish` command described [here](https://docs.airbyte.io/connector-development#updating-an-existing-connector)
+- [ ] After the new connector version is published, connector version bumped in the seed directory as described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)
+- [ ] Seed specs have been re-generated by building the platform and committing the changes to the seed spec files, as described [here](https://docs.airbyte.io/connector-development#publishing-a-connector)
 
 </p>
 </details>

.github/workflows/doc-link-check.json

@@ -45,11 +45,15 @@
       "reason": "Test only scaffold connector",
       "pattern": "destinations/scaffold-"
     },
+    {
+      "reason": "Returns a 403 for many valid pages",
+      "pattern": "https://mvnrepository.com/artifact/"
+    },
     {
       "reason": "Archived articles aren't actively maintained.",
       "pattern": "archive/"
     }
   ],
   "retryOn429": false,
-  "aliveStatusCodes": [200, 206, 401, 403, 429, 503]
+  "aliveStatusCodes": [200, 206, 429, 503, 0]
 }

.github/workflows/doc-link-check.yml

@@ -1,5 +1,6 @@
 # Perform link check on all markdown files
-name: Doc Link Checker (Full)
+
+name: Doc Link Checker
 
 on:
   push:
@@ -12,10 +13,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@master
+      # check all files on master
       - uses: gaurav-nelson/github-action-markdown-link-check@v1
+        if: github.ref == 'refs/heads/master'
         with:
-          use-quiet-mode: 'no'
-          use-verbose-mode: 'yes'
+          use-quiet-mode: 'yes'
           check-modified-files-only: 'no'
           config-file: .github/workflows/doc-link-check.json
-          base-branch: ${{ github.base_ref }}
+#      # check changed files for branches
+#      - uses: gaurav-nelson/github-action-markdown-link-check@v1
+#        if: github.ref != 'refs/heads/master'
+#        with:
+#          use-quiet-mode: 'yes'
+#          check-modified-files-only: 'yes'
+#          config-file: .github/workflows/doc-link-check.json

.github/workflows/gradle.yml

@@ -459,6 +459,17 @@ jobs:
       - name: Build Platform Docker Images
         run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
 
+      - name: Run Kubernetes End-to-End Acceptance Tests
+        env:
+          USER: root
+          HOME: /home/runner
+          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+          SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }}
+        run: |
+          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
+
       - name: Run Logging Tests
         run: ./tools/bin/cloud_storage_logging_test.sh
         env:
@@ -481,16 +492,6 @@ jobs:
         run: |
           CI=true ./tools/bin/gcp_acceptance_tests.sh
 
-      - name: Run Kubernetes End-to-End Acceptance Tests
-        env:
-          USER: root
-          HOME: /home/runner
-          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
-          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
-          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
-          SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }}
-        run: |
-          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
   # In case of self-hosted EC2 errors, remove this block.
   stop-kube-acceptance-test-runner:
     name: Stop Kube Acceptance Test EC2 Runner

.github/workflows/publish-command.yml

@@ -107,6 +107,7 @@ jobs:
           GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
           GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
           GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
+          GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
@@ -191,13 +192,17 @@ jobs:
           MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }}
           SOURCE_ONESIGNAL_TEST_CREDS: ${{ secrets.SOURCE_ONESIGNAL_TEST_CREDS }}
           SOURCE_SALESLOFT_TEST_CREDS: ${{ secrets.SOURCE_SALESLOFT_TEST_CREDS }}
+          SOURCE_CONFLUENCE_TEST_CREDS: ${{ secrets.SOURCE_CONFLUENCE_TEST_CREDS }}
           SOURCE_AMAZON_SQS_TEST_CREDS: ${{ secrets.SOURCE_AMAZON_SQS_TEST_CREDS }}
           SOURCE_FRESHSERVICE_TEST_CREDS: ${{ secrets.SOURCE_FRESHSERVICE_TEST_CREDS }}
           SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }}
           SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }}
           SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }}
           SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }}
           SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }}
+          SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }}
+          SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }}
+          SOURCE_MONDAY_TEST_CREDS: ${{ secrets.SOURCE_MONDAY_TEST_CREDS }}
       - run: |
           echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
           ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache

.github/workflows/test-command.yml

@@ -102,6 +102,7 @@ jobs:
           GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD: ${{ secrets.GOOGLE_ANALYTICS_V4_TEST_CREDS_OLD }}
           GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
           GOOGLE_DIRECTORY_TEST_CREDS: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS }}
+          GOOGLE_DIRECTORY_TEST_CREDS_OAUTH: ${{ secrets.GOOGLE_DIRECTORY_TEST_CREDS_OAUTH }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS }}
           GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC: ${{ secrets.GOOGLE_SEARCH_CONSOLE_CDK_TEST_CREDS_SRV_ACC }}
           GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }}
@@ -186,13 +187,17 @@ jobs:
           MONGODB_TEST_CREDS: ${{ secrets.MONGODB_TEST_CREDS }}
           SOURCE_ONESIGNAL_TEST_CREDS: ${{ secrets.SOURCE_ONESIGNAL_TEST_CREDS }}
           SOURCE_SALESLOFT_TEST_CREDS: ${{ secrets.SOURCE_SALESLOFT_TEST_CREDS }}
+          SOURCE_CONFLUENCE_TEST_CREDS: ${{ secrets.SOURCE_CONFLUENCE_TEST_CREDS }}
           SOURCE_AMAZON_SQS_TEST_CREDS: ${{ secrets.SOURCE_AMAZON_SQS_TEST_CREDS }}
           SOURCE_FRESHSERVICE_TEST_CREDS: ${{ secrets.SOURCE_FRESHSERVICE_TEST_CREDS }}
           SOURCE_LEMLIST_TEST_CREDS: ${{ secrets.SOURCE_LEMLIST_TEST_CREDS }}
           SOURCE_STRAVA_TEST_CREDS: ${{ secrets.SOURCE_STRAVA_TEST_CREDS }}
           SOURCE_PAYSTACK_TEST_CREDS: ${{ secrets.SOURCE_PAYSTACK_TEST_CREDS }}
           SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }}
           SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }}
+          SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }}
+          SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }}
+          SOURCE_MONDAY_TEST_CREDS: ${{ secrets.SOURCE_MONDAY_TEST_CREDS }}
       - run: |
           ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }}
         name: test ${{ github.event.inputs.connector }}

README.md

@@ -34,8 +34,6 @@ Now visit [http://localhost:8000](http://localhost:8000)
 
 Here is a [step-by-step guide](https://github.com/airbytehq/airbyte/tree/e378d40236b6a34e1c1cb481c8952735ec687d88/docs/quickstart/getting-started.md) showing you how to load data from an API into a file, all on your computer.
 
-If you want to schedule a 20-min call with our team to help you get set up, please select [some time directly here](https://calendly.com/nataliekwong/airbyte-onboarding).
-
 ## Features
 
 * **Built for extensibility**: Adapt an existing connector to your needs or build a new one with ease.

airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java

@@ -22,17 +22,17 @@ public class LineGobbler implements VoidCallable {
   private final static Logger LOGGER = LoggerFactory.getLogger(LineGobbler.class);
 
   public static void gobble(final InputStream is, final Consumer<String> consumer) {
-    gobble(is, consumer, "generic", MdcScope.DEFAULT);
+    gobble(is, consumer, "generic", MdcScope.DEFAULT_BUILDER);
   }
 
-  public static void gobble(final InputStream is, final Consumer<String> consumer, final MdcScope mdcScope) {
-    gobble(is, consumer, "generic", mdcScope);
+  public static void gobble(final InputStream is, final Consumer<String> consumer, final MdcScope.Builder mdcScopeBuilder) {
+    gobble(is, consumer, "generic", mdcScopeBuilder);
   }
 
-  public static void gobble(final InputStream is, final Consumer<String> consumer, final String caller, final MdcScope mdcScope) {
+  public static void gobble(final InputStream is, final Consumer<String> consumer, final String caller, final MdcScope.Builder mdcScopeBuilder) {
     final ExecutorService executor = Executors.newSingleThreadExecutor();
     final Map<String, String> mdc = MDC.getCopyOfContextMap();
-    final var gobbler = new LineGobbler(is, consumer, executor, mdc, caller, mdcScope);
+    final var gobbler = new LineGobbler(is, consumer, executor, mdc, caller, mdcScopeBuilder);
     executor.submit(gobbler);
   }
 
@@ -41,35 +41,35 @@ public static void gobble(final InputStream is, final Consumer<String> consumer,
   private final ExecutorService executor;
   private final Map<String, String> mdc;
   private final String caller;
-  private final MdcScope containerLogMDC;
+  private final MdcScope.Builder containerLogMdcBuilder;
 
   LineGobbler(final InputStream is,
               final Consumer<String> consumer,
               final ExecutorService executor,
               final Map<String, String> mdc) {
-    this(is, consumer, executor, mdc, "generic", MdcScope.DEFAULT);
+    this(is, consumer, executor, mdc, "generic", MdcScope.DEFAULT_BUILDER);
   }
 
   LineGobbler(final InputStream is,
               final Consumer<String> consumer,
               final ExecutorService executor,
               final Map<String, String> mdc,
-              final MdcScope mdcScope) {
-    this(is, consumer, executor, mdc, "generic", mdcScope);
+              final MdcScope.Builder mdcScopeBuilder) {
+    this(is, consumer, executor, mdc, "generic", mdcScopeBuilder);
   }
 
   LineGobbler(final InputStream is,
               final Consumer<String> consumer,
               final ExecutorService executor,
               final Map<String, String> mdc,
               final String caller,
-              final MdcScope mdcScope) {
+              final MdcScope.Builder mdcScopeBuilder) {
     this.is = IOs.newBufferedReader(is);
     this.consumer = consumer;
     this.executor = executor;
     this.mdc = mdc;
     this.caller = caller;
-    this.containerLogMDC = mdcScope;
+    this.containerLogMdcBuilder = mdcScopeBuilder;
   }
 
   @Override
@@ -78,7 +78,7 @@ public void voidCall() {
     try {
       String line;
       while ((line = is.readLine()) != null) {
-        try (containerLogMDC) {
+        try (final var mdcScope = containerLogMdcBuilder.build()) {
           consumer.accept(line);
         }
       }

airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java

@@ -17,7 +17,7 @@
  * <pre>
  *   <code>
  *     try(final ScopedMDCChange scopedMDCChange = new ScopedMDCChange(
- *      new HashMap<String, String>() {{
+ *      new HashMap&lt;String, String&gt;() {{
  *        put("my", "value");
  *      }}
  *     )) {
@@ -28,7 +28,7 @@
  */
 public class MdcScope implements AutoCloseable {
 
-  public final static MdcScope DEFAULT = new Builder().build();
+  public final static MdcScope.Builder DEFAULT_BUILDER = new Builder();
 
   private final Map<String, String> originalContextMap;
 

airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java

@@ -27,8 +27,8 @@ public static <T> AutoCloseableIterator<T> fromIterator(final Iterator<T> iterat
   }
 
   /**
-   * Coerces a vanilla {@link Iterator} into a {@link AutoCloseableIterator}. The provided
-   * {@param onClose} function will be called at most one time.
+   * Coerces a vanilla {@link Iterator} into a {@link AutoCloseableIterator}. The provided onClose
+   * function will be called at most one time.
    *
    * @param iterator autocloseable iterator to add another close to
    * @param onClose the function that will be called on close

airbyte-commons/src/main/resources/log4j2.xml

@@ -7,7 +7,7 @@
         <!-- The following patter log the application name in the beginning of the line, then a regular log line after a - separator. If the application name is not present in the context, the separator is removed -->
         <Property name="docker-worker-file-pattern">%replace{%X{log_source} - }{^ - }{}%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p (%X{job_root}) %C{1}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
         <!-- Remove paths from logs on Cloud as users aren't able to access file paths and these end up being noise. -->
-        <Property name="cloud-worker-file-pattern">%d{yyyy-MM-dd HH:mm:ss} %-5p %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
+        <Property name="cloud-worker-file-pattern">%replace{%X{log_source} - }{^ - }{}%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %p %C{1}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
 
         <!-- Always log INFO by default. -->
         <Property name="log-level">$${env:LOG_LEVEL:-INFO}</Property>

airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java

@@ -9,7 +9,6 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
@@ -18,6 +17,7 @@
 import io.airbyte.commons.concurrency.VoidCallable;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.Test;
 
@@ -38,15 +38,18 @@ void testFromIterator() throws Exception {
 
   @Test
   void testFromStream() throws Exception {
-    final Stream<String> stream = spy(Stream.of("a", "b", "c"));
+    final AtomicBoolean isClosed = new AtomicBoolean(false);
+    final Stream<String> stream = Stream.of("a", "b", "c");
+    stream.onClose(() -> isClosed.set(true));
+
     final AutoCloseableIterator<String> iterator = AutoCloseableIterators.fromStream(stream);
 
     assertNext(iterator, "a");
     assertNext(iterator, "b");
     assertNext(iterator, "c");
     iterator.close();
 
-    verify(stream).close();
+    assertTrue(isClosed.get());
   }
 
   private void assertNext(final Iterator<String> iterator, final String value) {

airbyte-config/init/Dockerfile

@@ -1,4 +1,5 @@
-FROM alpine:3.4 AS seed
+ARG ALPINE_IMAGE=alpine:3.4
+FROM ${ALPINE_IMAGE} AS seed
 
 WORKDIR /app
 

airbyte-config/init/build.gradle

@@ -6,4 +6,8 @@ dependencies {
     implementation 'commons-cli:commons-cli:1.4'
 
     implementation project(':airbyte-config:models')
+    implementation project(':airbyte-config:persistence')
+    implementation project(':airbyte-protocol:models')
+    implementation project(':airbyte-commons-docker')
+    implementation project(':airbyte-json-validation')
 }

airbyte-config/init/src/main/java/io/airbyte/config/init/SeedType.java

@@ -7,7 +7,9 @@
 public enum SeedType {
 
   STANDARD_SOURCE_DEFINITION("/seed/source_definitions.yaml", "sourceDefinitionId"),
-  STANDARD_DESTINATION_DEFINITION("/seed/destination_definitions.yaml", "destinationDefinitionId");
+  STANDARD_DESTINATION_DEFINITION("/seed/destination_definitions.yaml", "destinationDefinitionId"),
+  SOURCE_SPEC("/seed/source_specs.yaml", "dockerImage"),
+  DESTINATION_SPEC("/seed/destination_specs.yaml", "dockerImage");
 
   final String resourcePath;
   // ID field name