🐛 Source GitHub: fix bug with IssueEvents stream and add handling f…

…or rate limiting (#4708) * Few updates for GitHub source Set correct `cursor_field` for `IssueEvents` stream. Add rate limit handling. Add handling for 403 error. Add handling for 502 error. Co-authored-by: Eugene Kulak <[email protected]> Co-authored-by: Sherif A. Nada <[email protected]>
airbytehq · Jul 15, 2021 · b405c9c · b405c9c
1 parent baed7b4
commit b405c9c
Show file tree

Hide file tree

Showing 6 changed files with 207 additions and 77 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e",
   "name": "GitHub",
   "dockerRepository": "airbyte/source-github",
-  "dockerImageTag": "0.1.1",
-  "documentationUrl": "https://hub.docker.com/r/airbyte/source-github",
+  "dockerImageTag": "0.1.2",
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/github",
   "icon": "github.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -34,8 +34,8 @@
 - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   name: GitHub
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.1.1
-  documentationUrl: https://hub.docker.com/r/airbyte/source-github
+  dockerImageTag: 0.1.2
+  documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
 - sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1
   name: Microsoft SQL Server (MSSQL)

diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py
@@ -64,22 +64,23 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) ->
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         authenticator = TokenAuthenticator(token=config["access_token"], auth_method="token")
         full_refresh_args = {"authenticator": authenticator, "repository": config["repository"]}
-        incremental_args = {"authenticator": authenticator, "repository": config["repository"], "start_date": config["start_date"]}
+        incremental_args = {**full_refresh_args, "start_date": config["start_date"]}
+
         return [
             Assignees(**full_refresh_args),
-            Reviews(**full_refresh_args),
             Collaborators(**full_refresh_args),
-            Teams(**full_refresh_args),
-            IssueLabels(**full_refresh_args),
-            Releases(**incremental_args),
-            Events(**incremental_args),
             Comments(**incremental_args),
-            PullRequests(**incremental_args),
             CommitComments(**incremental_args),
-            IssueMilestones(**incremental_args),
             Commits(**incremental_args),
-            Stargazers(**incremental_args),
-            Projects(**incremental_args),
-            Issues(**incremental_args),
+            Events(**incremental_args),
             IssueEvents(**incremental_args),
+            IssueLabels(**full_refresh_args),
+            IssueMilestones(**incremental_args),
+            Issues(**incremental_args),
+            Projects(**incremental_args),
+            PullRequests(**incremental_args),
+            Releases(**incremental_args),
+            Reviews(**full_refresh_args),
+            Stargazers(**incremental_args),
+            Teams(**full_refresh_args),
         ]