🐛 Source RKI (Robert Koch-Institut): add message when no data availab…

…le (#15667) * updated parse_response with a message. * updated test and executed gradlew * updated unit tests. * applied gradlew with no daemon for airbyte python format * change addProperties to true * update docs Co-authored-by: marcosmarxm <[email protected]>
airbytehq · Aug 25, 2022 · 2c89185 · 2c89185
1 parent c402264
commit 2c89185
Show file tree

Hide file tree

Showing 10 changed files with 68 additions and 33 deletions.
diff --git a/airbyte-integrations/connectors/source-rki-covid/Dockerfile b/airbyte-integrations/connectors/source-rki-covid/Dockerfile
@@ -30,5 +30,5 @@ COPY source_rki_covid ./source_rki_covid
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-rki-covid
diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py
@@ -109,7 +109,7 @@ def cursor_field(self) -> str:
 
     def date_to_int(self, start_date) -> int:
         diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-        if diff.days == 0:
+        if diff.days <= 0:
             return 1
         return diff.days
 
@@ -128,7 +128,9 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter
             yield from records
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        return response.json().get("data")
+        if response.json().get("data"):
+            return response.json().get("data")
+        return [{}]
 
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -158,7 +160,7 @@ def cursor_field(self) -> str:
 
     def date_to_int(self, start_date) -> int:
         diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-        if diff.days == 0:
+        if diff.days <= 0:
             return 1
         return diff.days
 
@@ -179,7 +181,7 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
         if response.json().get("data"):
             return response.json().get("data")
-        pass
+        return [{}]
 
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -209,7 +211,7 @@ def cursor_field(self) -> str:
 
     def date_to_int(self, start_date) -> int:
         diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-        if diff.days == 0:
+        if diff.days <= 0:
             return 1
         return diff.days
 
@@ -228,7 +230,9 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter
             yield from records
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        return response.json().get("data")
+        if response.json().get("data"):
+            return response.json().get("data")
+        return [{}]
 
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -258,7 +262,7 @@ def cursor_field(self) -> str:
 
     def date_to_int(self, start_date) -> int:
         diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-        if diff.days == 0:
+        if diff.days <= 0:
             return 1
         return diff.days
 
@@ -277,7 +281,9 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter
             yield from records
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        return response.json().get("data")
+        if response.json().get("data"):
+            return response.json().get("data")
+        return [{}]
 
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -307,7 +313,7 @@ def cursor_field(self) -> str:
 
     def date_to_int(self, start_date) -> int:
         diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-        if diff.days == 0:
+        if diff.days <= 0:
             return 1
         return diff.days
 
@@ -326,7 +332,9 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter
             yield from records
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        return response.json().get("data").get("history")
+        if response.json().get("data"):
+            return response.json().get("data").get("history")
+        return [{}]
 
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
@@ -356,7 +364,7 @@ def cursor_field(self) -> str:
 
     def date_to_int(self, start_date) -> int:
         diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-        if diff.days == 0:
+        if diff.days <= 0:
             return 1
         return diff.days
 
@@ -375,7 +383,9 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter
             yield from records
 
     def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
-        return response.json().get("data")
+        if response.json().get("data"):
+            return response.json().get("data")
+        return [{}]
 
     def path(
         self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None

diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/spec.json
@@ -5,7 +5,7 @@
     "title": "RKI Covid Spec",
     "type": "object",
     "required": ["start_date"],
-    "additionalProperties": false,
+    "additionalProperties": true,
     "properties": {
       "start_date": {
         "type": "string",

diff --git a/...ntegrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py b/...ntegrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorycases.py
@@ -12,7 +12,7 @@
 
 @fixture
 def patch_incremental_german_history_cases(mocker):
-    # Mock abstract methods to enable instantiating abstract class
+    # Mock abstract methods to enable instantiating abstract class.
     mocker.patch.object(GermanyHistoryCases, "primary_key", None)
 
 
@@ -27,13 +27,17 @@ def test_parse_response(patch_incremental_german_history_cases):
     config = {"start_date": "2022-04-27"}
     stream = GermanyHistoryCases(config)
     response = requests.get("https://api.corona-zahlen.org/germany/history/cases/1")
-    expected_response = response.json().get("data")
-    assert stream.parse_response(response) == expected_response
+    if response.json().get("data"):
+        expected_response = response.json().get("data")
+        assert stream.parse_response(response) == expected_response
+    else:
+        expected_response = [{}]
+        assert stream.parse_response(response) == expected_response
 
 
 def check_diff(start_date):
     diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-    if diff.days == 0:
+    if diff.days <= 0:
         return str(1)
     return str(diff.days)
 

diff --git a/...tegrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py b/...tegrations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistorydeaths.py
@@ -37,13 +37,17 @@ def test_parse_response(patch_incremental_german_history_deaths):
     config = {"start_date": "2022-04-27"}
     stream = GermanHistoryDeaths(config)
     response = requests.get("https://api.corona-zahlen.org/germany/history/deaths/1")
-    expected_response = response.json().get("data")
-    assert stream.parse_response(response) == expected_response
+    if response.json().get("data"):
+        expected_response = response.json().get("data")
+        assert stream.parse_response(response) == expected_response
+    else:
+        expected_response = [{}]
+        assert stream.parse_response(response) == expected_response
 
 
 def check_diff(start_date):
     diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-    if diff.days == 0:
+    if diff.days <= 0:
         return str(1)
     return str(diff.days)
 

diff --git a/...s/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py b/...s/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryfrozenIncidence.py
@@ -37,13 +37,17 @@ def test_parse_response(patch_incremental_german_history_frozenInc):
     config = {"start_date": "2022-04-27"}
     stream = GermanHistoryFrozenIncidence(config)
     response = requests.get("https://api.corona-zahlen.org/germany/history/frozen-incidence/1")
-    expected_response = response.json().get("data").get("history")
-    assert stream.parse_response(response) == expected_response
+    if response.json().get("data").get("history"):
+        expected_response = response.json().get("data").get("history")
+        assert stream.parse_response(response) == expected_response
+    else:
+        expected_response = []
+        assert stream.parse_response(response) == expected_response
 
 
 def check_diff(start_date):
     diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-    if diff.days == 0:
+    if diff.days <= 0:
         return str(1)
     return str(diff.days)
 

diff --git a/...s/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py b/...s/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryhospitalization.py
@@ -37,13 +37,17 @@ def test_parse_response(patch_incremental_german_history_hospitalization):
     config = {"start_date": "2022-04-27"}
     stream = GermanHistoryHospitalization(config)
     response = requests.get("https://api.corona-zahlen.org/germany/history/hospitalization/1")
-    expected_response = response.json().get("data")
-    assert stream.parse_response(response) == expected_response
+    if response.json().get("data"):
+        expected_response = response.json().get("data")
+        assert stream.parse_response(response) == expected_response
+    else:
+        expected_response = [{}]
+        assert stream.parse_response(response) == expected_response
 
 
 def check_diff(start_date):
     diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-    if diff.days == 0:
+    if diff.days <= 0:
         return str(1)
     return str(diff.days)
 

diff --git a/...rations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py b/...rations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryincidence.py
@@ -37,13 +37,17 @@ def test_parse_response(patch_incremental_german_history_incidence):
     config = {"start_date": "2022-04-27"}
     stream = GermanHistoryIncidence(config)
     response = requests.get("https://api.corona-zahlen.org/germany/history/incidence/1")
-    expected_response = response.json().get("data")
-    assert stream.parse_response(response) == expected_response
+    if response.json().get("data"):
+        expected_response = response.json().get("data")
+        assert stream.parse_response(response) == expected_response
+    else:
+        expected_response = [{}]
+        assert stream.parse_response(response) == expected_response
 
 
 def check_diff(start_date):
     diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-    if diff.days == 0:
+    if diff.days <= 0:
         return str(1)
     return str(diff.days)
 

diff --git a/...rations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py b/...rations/connectors/source-rki-covid/unit_tests/test_incremental_germanhistoryrecovered.py
@@ -37,13 +37,17 @@ def test_parse_response(patch_incremental_german_history_recovered):
     config = {"start_date": "2022-04-27"}
     stream = GermanHistoryRecovered(config)
     response = requests.get("https://api.corona-zahlen.org/germany/history/recovered/1")
-    expected_response = response.json().get("data")
-    assert stream.parse_response(response) == expected_response
+    if response.json().get("data"):
+        expected_response = response.json().get("data")
+        assert stream.parse_response(response) == expected_response
+    else:
+        expected_response = [{}]
+        assert stream.parse_response(response) == expected_response
 
 
 def check_diff(start_date):
     diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d")
-    if diff.days == 0:
+    if diff.days <= 0:
         return str(1)
     return str(diff.days)
 

diff --git a/docs/integrations/sources/rki-covid.md b/docs/integrations/sources/rki-covid.md
@@ -53,5 +53,6 @@ Select start date
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.2 | 2022-08-25 | [15667](https://github.com/airbytehq/airbyte/pull/15667) | Add message when no data available |
 | 0.1.1 | 2022-05-30 | [11732](https://github.com/airbytehq/airbyte/pull/11732) | Fix docs |
 | 0.1.0 | 2022-05-30 | [11732](https://github.com/airbytehq/airbyte/pull/11732) | Initial Release |