Source GitHub: #970 handle 502 Bad Gateway (#21481)

* Source Github: handle 502 Bad Gateway with proper log message * Source Github: bump version * Source Github: update changelog for new 0.3.12 version * auto-bump connector version Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Jan 19, 2023 · 3cebbbb · 3cebbbb
1 parent 540ceb0
commit 3cebbbb
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 1 deletion.
diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile
@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.11
+LABEL io.airbyte.version=0.3.12
 LABEL io.airbyte.name=airbyte/source-github
diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py
@@ -171,6 +171,8 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter
                 )
             elif e.response.status_code == requests.codes.SERVER_ERROR and isinstance(self, WorkflowRuns):
                 error_msg = f"Syncing `{self.name}` stream isn't available for repository `{stream_slice['repository']}`."
+            elif e.response.status_code == requests.codes.BAD_GATEWAY:
+                error_msg = f"Stream {self.name} temporary failed. Try to re-run sync later"
             else:
                 # most probably here we're facing a 500 server error and a risk to get a non-json response, so lets output response.text
                 self.logger.error(f"Undefined error while reading records: {e.response.text}")

diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py
@@ -189,6 +189,26 @@ def test_stream_teams_404():
     assert responses.calls[0].request.url == "https://api.github.com/orgs/org_name/teams?per_page=100"
 
 
+@responses.activate
+@patch("time.sleep")
+def test_stream_teams_502(sleep_mock):
+    organization_args = {"organizations": ["org_name"]}
+    stream = Teams(**organization_args)
+
+    url = "https://api.github.com/orgs/org_name/teams"
+    responses.add(
+        method="GET",
+        url=url,
+        status=requests.codes.BAD_GATEWAY,
+        json={"message": "Server Error"},
+    )
+
+    assert list(read_full_refresh(stream)) == []
+    assert len(responses.calls) == 6
+    # Check whether url is the same for all response.calls
+    assert set(call.request.url for call in responses.calls).symmetric_difference({f"{url}?per_page=100"}) == set()
+
+
 @responses.activate
 def test_stream_organizations_read():
     organization_args = {"organizations": ["org1", "org2"]}

diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md
@@ -163,6 +163,7 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.3.12  | 2023-01-18 | [21481](https://github.com/airbytehq/airbyte/pull/21481)                                                          | Handle 502 Bad Gateway error with proper log message                                                                                                 |
 | 0.3.11  | 2023-01-06 | [21084](https://github.com/airbytehq/airbyte/pull/21084)                                                          | Raise Error if no organizations or repos are available during read                                                                                                  |
 | 0.3.10  | 2022-12-15 | [20523](https://github.com/airbytehq/airbyte/pull/20523)                                                          | Revert changes from 0.3.9                                                                                                                                           |
 | 0.3.9   | 2022-12-14 | [19978](https://github.com/airbytehq/airbyte/pull/19978)                                                          | Update CDK dependency; move custom HTTPError handling into `AvailabilityStrategy` classes                                                                           |