Update source-gnews manifest to use inline stream schemas

airbytehq · Dec 13, 2022 · f326718 · f326718
1 parent 4b4c769
commit f326718
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 90 deletions.
diff --git a/airbyte-integrations/connectors/source-gnews/setup.py b/airbyte-integrations/connectors/source-gnews/setup.py
@@ -6,7 +6,7 @@
 from setuptools import find_packages, setup
 
 MAIN_REQUIREMENTS = [
-    "airbyte-cdk~=0.1",
+    "airbyte-cdk~=0.13.2",
 ]
 
 TEST_REQUIREMENTS = [

diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml b/airbyte-integrations/connectors/source-gnews/source_gnews/gnews.yaml
@@ -1,5 +1,107 @@
 version: "0.1.0"
 
+schemas:
+  search_stream_schema:
+    type: object
+    properties:
+      title:
+        type:
+          - "null"
+          - string
+        description: The main title of the article.
+      description:
+        type:
+          - "null"
+          - string
+        description: The small paragraph under the title.
+      content:
+        type:
+          - "null"
+          - string
+        description: All the content of the article.
+      url:
+        type:
+          - "null"
+          - string
+        description: The URL of the article.
+      image:
+        type:
+          - "null"
+          - string
+        description: The main image of the article.
+      publishedAt:
+        type:
+          - "null"
+          - string
+        description:
+          The date of publication of the article. The date is always in the
+          UTC time zone.
+      source:
+        type:
+          - "null"
+          - object
+        properties:
+          name:
+            type:
+              - "null"
+              - string
+            description: The name of the source.
+          url:
+            type:
+              - "null"
+              - string
+            description: The home page of the source.
+  top_headlines_stream_schema:
+    type: object
+    properties:
+      title:
+        type:
+          - "null"
+          - string
+        description: The main title of the article.
+      description:
+        type:
+          - "null"
+          - string
+        description: The small paragraph under the title.
+      content:
+        type:
+          - "null"
+          - string
+        description: All the content of the article.
+      url:
+        type:
+          - "null"
+          - string
+        description: The URL of the article.
+      image:
+        type:
+          - "null"
+          - string
+        description: The main image of the article.
+      publishedAt:
+        type:
+          - "null"
+          - string
+        description:
+          The date of publication of the article. The date is always in the
+          UTC time zone.
+      source:
+        type:
+          - "null"
+          - object
+        properties:
+          name:
+            type:
+              - "null"
+              - string
+            description: The name of the source.
+          url:
+            type:
+              - "null"
+              - string
+            description: The home page of the source.
+
 definitions:
   selector:
     extractor:
@@ -44,7 +146,12 @@ definitions:
     nullable: "{{ ','.join(config['nullable']) }}"
     from: "{{ stream_slice['start_time'] }}"
     to: "{{ stream_slice['end_time'] }}"
+  schema_loader:
+    type: StaticSchemaLoader
   search_stream:
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+      schema: "*ref(schemas.search_stream_schema)"
     $options:
       name: "search"
       primary_key: "url"
@@ -61,6 +168,9 @@ definitions:
             in: "{{ ','.join(config['in']) }}"
             sortby: "{{ config['sortby'] }}"
   top_headlines_stream:
+    schema_loader:
+      $ref: "*ref(definitions.schema_loader)"
+      schema: "*ref(schemas.top_headlines_stream_schema)"
     $options:
       name: "top_headlines"
       primary_key: "url"

diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/search.json b/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/search.json
diff --git a/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/top_headlines.json b/airbyte-integrations/connectors/source-gnews/source_gnews/schemas/top_headlines.json
diff --git a/docs/integrations/sources/gnews.md b/docs/integrations/sources/gnews.md
@@ -35,6 +35,7 @@ Rate Limiting is based on the API Key tier subscription, get more info [here](ht
 
 ## Changelog
 
-| Version | Date       | Pull Request                                             | Subject                                         |
-|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------|
-| 0.1.0   | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews                            |
+| Version | Date       | Pull Request                                             | Subject                                          |
+|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------|
+| 0.1.1   | 2022-12-13 | [20405](https://github.com/airbytehq/airbyte/pull/20405) | Update the manifest to use inline stream schemas |
+| 0.1.0   | 2022-11-01 | [18808](https://github.com/airbytehq/airbyte/pull/18808) | 🎉 New Source: GNews                             |