Source Sentry: add state persistence (#21864)

* Source Sentry: add state persistance * Source Sentry: bump version * Source Sentry: update changelog * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Jan 26, 2023 · 795dfe9 · 795dfe9
1 parent 862c946
commit 795dfe9
Show file tree

Hide file tree

Showing 6 changed files with 26 additions and 10 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -2029,7 +2029,7 @@
 - sourceDefinitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781
   name: Sentry
   dockerRepository: airbyte/source-sentry
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.com/integrations/sources/sentry
   icon: sentry.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -16631,7 +16631,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-sentry:0.1.8"
+- dockerImage: "airbyte/source-sentry:0.1.9"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/sentry"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile
@@ -34,5 +34,5 @@ COPY source_sentry ./source_sentry
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-sentry
diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl
diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py
@@ -65,14 +65,23 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
 
 
 class SentryIncremental(SentryStreamPagination, IncrementalMixin):
+    def __init__(self, *args, **kwargs):
+        super(SentryIncremental, self).__init__(*args, **kwargs)
+        self._cursor_value = None
+
     def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable:
         """
         Endpoint does not provide query filtering params, but they provide us
         cursor field in most cases, so we used that as incremental filtering
         during the parsing.
         """
         start_date = "1900-01-01T00:00:00.0Z"
-        if pendulum.parse(record[self.cursor_field]) >= pendulum.parse((stream_state or {}).get(self.cursor_field, start_date)):
+        if pendulum.parse(record[self.cursor_field]) > pendulum.parse((stream_state or {}).get(self.cursor_field, start_date)):
+            # Persist state.
+            # There is a bug in state setter: because of self._cursor_value is not defined it raises Attribute error
+            # which is ignored in airbyte_cdk/sources/abstract_source.py:320 and we have an empty state in return
+            # See: https://github.com/airbytehq/oncall/issues/1317
+            self.state = record
             yield record
 
     def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[MutableMapping]:
@@ -87,7 +96,13 @@ def state(self) -> Mapping[str, Any]:
 
     @state.setter
     def state(self, value: Mapping[str, Any]):
-        self._cursor_value = value[self.cursor_field]
+        """
+        Define state as a max between given value and current state
+        """
+        if not self._cursor_value:
+            self._cursor_value = value[self.cursor_field]
+        else:
+            self._cursor_value = max(value[self.cursor_field], self.state[self.cursor_field])
 
 
 class Events(SentryIncremental):

diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md
@@ -45,6 +45,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air
 
 | Version | Date       | Pull Request                                             | Subject                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------|
+| 0.1.9   | 2022-12-20 | [21864](https://github.com/airbytehq/airbyte/pull/21864) | Add state persistence to incremental sync         |
 | 0.1.8   | 2022-12-20 | [20709](https://github.com/airbytehq/airbyte/pull/20709) | Add incremental sync                              |
 | 0.1.7   | 2022-09-30 | [17466](https://github.com/airbytehq/airbyte/pull/17466) | Migrate to per-stream states                      |
 | 0.1.6   | 2022-08-29 | [16112](https://github.com/airbytehq/airbyte/pull/16112) | Revert back to the Python CDK                     |