source-airtable: improve discovery logic (#18491)

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -9,7 +9,7 @@
 - name: Airtable
   sourceDefinitionId: 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212
   dockerRepository: airbyte/source-airtable
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.3
   documentationUrl: https://docs.airbyte.com/integrations/sources/airtable
   icon: airtable.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -156,7 +156,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-airtable:0.1.2"
+- dockerImage: "airbyte/source-airtable:0.1.3"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/airtable"
     connectionSpecification:
@@ -167,7 +167,6 @@
       - "api_key"
       - "base_id"
       - "tables"
-      additionalProperties: false
       properties:
         api_key:
           type: "string"

airbyte-integrations/connectors/source-airtable/Dockerfile

@@ -34,5 +34,5 @@ COPY source_airtable ./source_airtable
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-airtable

airbyte-integrations/connectors/source-airtable/source_airtable/helpers.py

@@ -13,8 +13,8 @@
 
 class Helpers(object):
     @staticmethod
-    def get_first_row(auth: TokenAuthenticator, base_id: str, table: str) -> Dict[str, Any]:
-        url = f"https://api.airtable.com/v0/{base_id}/{table}?pageSize=1"
+    def get_most_complete_row(auth: TokenAuthenticator, base_id: str, table: str, sample_size: int = 100) -> Dict[str, Any]:
+        url = f"https://api.airtable.com/v0/{base_id}/{table}?pageSize={sample_size}"
         try:
             response = requests.get(url, headers=auth.get_auth_header())
             response.raise_for_status()
@@ -26,8 +26,12 @@ def get_first_row(auth: TokenAuthenticator, base_id: str, table: str) -> Dict[st
             else:
                 raise Exception(f"Error getting first row from table {table}: {e}")
         json_response = response.json()
-        record = json_response.get("records", [])[0]
-        return record
+        records = json_response.get("records", [])
+        most_complete_row = records[0]
+        for record in records:
+            if len(record.keys()) > len(most_complete_row.keys()):
+                most_complete_row = record
+        return most_complete_row
 
     @staticmethod
     def get_json_schema(record: Dict[str, Any]) -> Dict[str, str]:

airbyte-integrations/connectors/source-airtable/source_airtable/source.py

@@ -75,7 +75,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
         auth = TokenAuthenticator(token=config["api_key"])
         for table in config["tables"]:
             try:
-                Helpers.get_first_row(auth, config["base_id"], table)
+                Helpers.get_most_complete_row(auth, config["base_id"], table)
             except Exception as e:
                 return False, str(e)
         return True, None
@@ -84,7 +84,7 @@ def discover(self, logger: AirbyteLogger, config) -> AirbyteCatalog:
         streams = []
         auth = TokenAuthenticator(token=config["api_key"])
         for table in config["tables"]:
-            record = Helpers.get_first_row(auth, config["base_id"], table)
+            record = Helpers.get_most_complete_row(auth, config["base_id"], table)
             json_schema = Helpers.get_json_schema(record)
             airbyte_stream = Helpers.get_airbyte_stream(table, json_schema)
             streams.append(airbyte_stream)
@@ -94,7 +94,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         auth = TokenAuthenticator(token=config["api_key"])
         streams = []
         for table in config["tables"]:
-            record = Helpers.get_first_row(auth, config["base_id"], table)
+            record = Helpers.get_most_complete_row(auth, config["base_id"], table)
             json_schema = Helpers.get_json_schema(record)
             stream = AirtableStream(base_id=config["base_id"], table_name=table, authenticator=auth, schema=json_schema)
             streams.append(stream)

airbyte-integrations/connectors/source-airtable/source_airtable/spec.json

@@ -5,7 +5,6 @@
     "title": "Airtable Source Spec",
     "type": "object",
     "required": ["api_key", "base_id", "tables"],
-    "additionalProperties": false,
     "properties": {
       "api_key": {
         "type": "string",

airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py

@@ -48,24 +48,24 @@ def expected_json_schema():
     }
 
 
-def test_get_first_row(auth, base_id, table, json_response):
+def test_get_most_complete_row(auth, base_id, table, json_response):
     with patch("requests.get") as mock_get:
         mock_get.return_value.status_code = HTTPStatus.OK
         mock_get.return_value.json.return_value = json_response
-        assert Helpers.get_first_row(auth, base_id, table) == {"id": "abc", "fields": {"name": "test"}}
+        assert Helpers.get_most_complete_row(auth, base_id, table) == {"id": "abc", "fields": {"name": "test"}}
 
 
-def test_get_first_row_invalid_api_key(base_id, table):
+def test_get_most_complete_row_invalid_api_key(base_id, table):
     with pytest.raises(Exception):
         auth = TokenAuthenticator("invalid_api_key")
-        Helpers.get_first_row(auth, base_id, table)
+        Helpers.get_most_complete_row(auth, base_id, table)
 
 
-def test_get_first_row_table_not_found(auth, base_id, table):
+def test_get_most_complete_row_table_not_found(auth, base_id, table):
     with patch("requests.exceptions.HTTPError") as mock_get:
         mock_get.return_value.status_code = HTTPStatus.NOT_FOUND
         with pytest.raises(Exception):
-            Helpers.get_first_row(auth, base_id, table)
+            Helpers.get_most_complete_row(auth, base_id, table)
 
 
 def test_get_json_schema(json_response, expected_json_schema):

airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py

@@ -18,11 +18,11 @@ def test_spec(config):
 
 def test_discover(config, mocker):
     source = SourceAirtable()
-    logger_mock, Helpers.get_first_row = MagicMock(), MagicMock()
+    logger_mock, Helpers.get_most_complete_row = MagicMock(), MagicMock()
     airbyte_catalog = source.discover(logger_mock, config)
     assert [stream.name for stream in airbyte_catalog.streams] == config["tables"]
     assert isinstance(airbyte_catalog, AirbyteCatalog)
-    assert Helpers.get_first_row.call_count == 2
+    assert Helpers.get_most_complete_row.call_count == 2
 
 
 @patch("requests.get")
@@ -34,7 +34,7 @@ def test_check_connection(config):
 
 def test_streams(config):
     source = SourceAirtable()
-    Helpers.get_first_row = MagicMock()
+    Helpers.get_most_complete_row = MagicMock()
     streams = source.streams(config)
     assert len(streams) == 2
     assert [stream.name for stream in streams] == config["tables"]

docs/integrations/sources/airtable.md

@@ -2,10 +2,10 @@
 
 ## Features
 
-| Feature | Supported?\(Yes/No\) | Notes |
-| :--- | :--- | :--- |
-| Full Refresh Sync | Yes |  |
-| Incremental Sync | No |  |
+| Feature           | Supported?\(Yes/No\) | Notes |
+| :---------------- | :------------------- | :---- |
+| Full Refresh Sync | Yes                  |       |
+| Incremental Sync  | No                   |       |
 
 This source syncs data from the [Airtable API](https://airtable.com/api).
 
@@ -33,7 +33,8 @@ See information about rate limits [here](https://support.airtable.com/hc/en-us/a
 
 ## Changelog
 
-| Version | Date | Pull Request | Subject |
-|:--------| :--------  | :-----       | :------ |
-| 0.1.2   | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy                                                             |
-| 0.1.1   | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec |
+| Version | Date       | Pull Request                                             | Subject                                  |
+| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------- |
+| 0.1.3   | 2022-10-26 | [18491](https://github.com/airbytehq/airbyte/pull/18491) | Improve schema discovery logic           |
+| 0.1.2   | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy         |
+| 0.1.1   | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425)   | Update title, description fields in spec |