Merge branch 'master' into grubberr/11417-source-amazon-ads

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.67-alpha
+current_version = 0.36.0-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -47,3 +47,7 @@ serialize =
 [bumpversion:file:octavia-cli/README.md]
 
 [bumpversion:file:octavia-cli/Dockerfile]
+
+[bumpversion:file:octavia-cli/setup.py]
+serialize = 
+	{major}.{minor}.{patch}

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.35.67-alpha
+VERSION=0.36.0-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/workflows/gradle.yml

@@ -146,26 +146,6 @@ jobs:
       - name: Run integration tests
         run: ./tools/bin/integration_tests_octavia.sh
 
-      - name: Slack Notification - Failure
-        if: failure() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildozer
-          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
-          SLACK_COLOR: DC143C
-          SLACK_TITLE: "Build failure"
-          SLACK_FOOTER: ""
-
-      - name: Slack Notification - Success
-        if: success() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildbot
-          SLACK_TITLE: "Build Success"
-          SLACK_FOOTER: ""
-
   # Connectors Base
   # In case of self-hosted EC2 errors, remove this block.
   start-connectors-base-build-runner:
@@ -248,25 +228,6 @@ jobs:
       - name: Ensure no file change
         run: git --no-pager diff && test -z "$(git --no-pager diff)"
 
-      - name: Slack Notification - Failure
-        if: failure() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildozer
-          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
-          SLACK_COLOR: DC143C
-          SLACK_TITLE: "Build failure"
-          SLACK_FOOTER: ""
-
-      - name: Slack Notification - Success
-        if: success() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildbot
-          SLACK_TITLE: "Build Success"
-          SLACK_FOOTER: ""
   # In case of self-hosted EC2 errors, remove this block.
   stop-connectors-base-build-runner:
     name: "Connectors Base: Stop Build EC2 Runner"
@@ -528,25 +489,6 @@ jobs:
       - name: Automatic Migration Acceptance Test
         run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
 
-      - name: Slack Notification - Failure
-        if: failure() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildozer
-          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
-          SLACK_COLOR: DC143C
-          SLACK_TITLE: "Build failure"
-          SLACK_FOOTER: ""
-
-      - name: Slack Notification - Success
-        if: success() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildbot
-          SLACK_TITLE: "Build Success"
-          SLACK_FOOTER: ""
   # In case of self-hosted EC2 errors, remove this block.
   stop-platform-build-runner:
     name: "Platform: Stop Build EC2 Runner"
@@ -655,25 +597,6 @@ jobs:
       - name: Run End-to-End Acceptance Tests with the new scheduler
         run: ./tools/bin/acceptance_test_with_new_scheduler.sh
 
-      - name: Slack Notification - Failure
-        if: failure() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildozer
-          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
-          SLACK_COLOR: DC143C
-          SLACK_TITLE: "Build failure"
-          SLACK_FOOTER: ""
-
-      - name: Slack Notification - Success
-        if: success() && github.ref == 'refs/heads/master'
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
-          SLACK_USERNAME: Buildbot
-          SLACK_TITLE: "Build Success"
-          SLACK_FOOTER: ""
   # In case of self-hosted EC2 errors, remove this block.
   stop-platform-new-scheduler-acceptance-runner:
     name: "Platform: Stop Docker w/ Scheduler v2 Test Runner"
@@ -1005,3 +928,62 @@ jobs:
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
           label: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner-v2.outputs.ec2-instance-id }}
+
+  notify-failure-slack-channel:
+    name: "Notify Slack Channel on Build Failures"
+    runs-on: ubuntu-latest
+    needs:
+      - build-connectors-base
+      - frontend-build
+      - octavia-cli-build
+      - platform-build
+      - platform-new-scheduler-acceptance
+      - kube-acceptance-test
+      - kube-acceptance-test-v2
+    if: ${{ failure() && github.ref == 'refs/heads/master' }}
+    steps:
+      - name: Publish to OSS Build Failure Slack Channel
+        uses: abinoda/slack-action@master
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          args: >-
+            {\"channel\":\"C03BEADRPNY\", \"blocks\":[
+            {\"type\":\"divider\"},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
+            {\"type\":\"divider\"}]}
+
+  notify-failure-slack-channel-fixed-broken-build:
+    name: "Notify Slack Channel on Build Fixes"
+    runs-on: ubuntu-latest
+    needs:
+      - build-connectors-base
+      - frontend-build
+      - octavia-cli-build
+      - platform-build
+      - platform-new-scheduler-acceptance
+      - kube-acceptance-test
+      - kube-acceptance-test-v2
+    if: success()
+    steps:
+      - name: Get Previous Workflow Status
+        uses: Mercymeilya/[email protected]
+        id: last_status
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+      # To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
+      - name: Publish Build Fixed Message to OSS Build Failure Slack Channel
+        if: ${{ steps.last_status.outputs.last_status == 'failure' }}
+        uses: abinoda/slack-action@master
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          args: >-
+            {\"channel\":\"C03AXC61YBZ\", \"blocks\":[
+            {\"type\":\"divider\"},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
+            {\"type\":\"divider\"}]}

.github/workflows/publish-cdk-command.yml

@@ -22,8 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        # 3.7.1 - is a minimal of 3.7.X version supported by github actions
-        python-version: [3.7.1, 3.7, 3.8, 3.9]
+        python-version: ["3.9"]
     steps:
       - uses: actions/setup-python@v2
         with:

airbyte-bootloader/Dockerfile

@@ -1,7 +1,7 @@
 ARG JDK_VERSION=17.0.1
 FROM openjdk:${JDK_VERSION}-slim
 
-ARG VERSION=0.35.67-alpha
+ARG VERSION=0.36.0-alpha
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.1.54
+- Add ability to import `IncrementalMixin` from `airbyte_cdk.sources.streams`.
+- Bumped minimum supported Python version to 3.9.
+
 ## 0.1.53
 Remove a false positive error logging during the send process.
 

airbyte-cdk/python/airbyte_cdk/sources/streams/__init__.py

@@ -3,6 +3,6 @@
 #
 
 # Initialize Streams Package
-from .core import Stream
+from .core import IncrementalMixin, Stream
 
-__all__ = ["Stream"]
+__all__ = ["IncrementalMixin", "Stream"]

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.53",
+    version="0.1.54",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",
@@ -33,8 +33,6 @@
         "Topic :: Software Development :: Libraries :: Python Modules",
         "License :: OSI Approved :: MIT License",
         # Python Version Support
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
     ],
     keywords="airbyte connector-development-kit cdk",

airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java

@@ -42,15 +42,15 @@
  * returning a list for query results. In addition, we provide helper functions that will just
  * return a single value (see: {@link JsonPaths#getSingleValue(JsonNode, String)}). These should
  * only be used if it is not possible for a query to return more than one value.
- *
- * Note: Package private as most uses of JsonPaths seems like they can be hidden inside other
- * commons libraries (i.e. Jsons and JsonsSchemas). If this assumption proves incorrect, we can open
- * it up.
  */
-class JsonPaths {
+public class JsonPaths {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(JsonPaths.class);
 
+  static final String JSON_PATH_START_CHARACTER = "$";
+  static final String JSON_PATH_LIST_SPLAT = "[*]";
+  static final String JSON_PATH_FIELD_SEPARATOR = ".";
+
   // set default configurations at start up to match our JSON setup.
   static {
     Configuration.setDefaults(new Configuration.Defaults() {
@@ -82,6 +82,18 @@ public Set<Option> options() {
     });
   }
 
+  public static String empty() {
+    return JSON_PATH_START_CHARACTER;
+  }
+
+  public static String appendField(final String jsonPath, final String field) {
+    return jsonPath + JSON_PATH_FIELD_SEPARATOR + field;
+  }
+
+  public static String appendAppendListSplat(final String jsonPath) {
+    return jsonPath + JSON_PATH_LIST_SPLAT;
+  }
+
   /*
    * This version of the JsonPath Configuration object allows queries to return to the path of values
    * instead of the values that were found.