From ff70fd390ed445a8a56bde4d29877a626acc2e64 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Tue, 31 Aug 2021 13:43:09 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Slack:=20Fix=20max=20re?= =?UTF-8?q?tries=20issue=20(#5697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Increased max retries to 10 * Add docs * Increased timeout seconds for acceptance tests * Fixed for pass tests * Updated to default backoff * Add sample config --- .../c2281cee-86f9-4a86-bb48-d23286b4c7bd.json | 2 +- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-slack/Dockerfile | 2 +- .../connectors/source-slack/acceptance-test-config.yml | 4 +++- .../integration_tests/full_refresh_catalog.json | 9 --------- .../source-slack/integration_tests/invalid_config.json | 5 ++++- .../source-slack/integration_tests/sample_config.json | 6 ++++++ .../connectors/source-slack/source_slack/source.py | 10 ---------- docs/integrations/sources/slack.md | 1 + 9 files changed, 17 insertions(+), 24 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json mode change 100644 => 100755 airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json create mode 100755 airbyte-integrations/connectors/source-slack/integration_tests/sample_config.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json index 7efce51a3b9b..cb2235a03f37 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c2281cee-86f9-4a86-bb48-d23286b4c7bd.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "c2281cee-86f9-4a86-bb48-d23286b4c7bd", "name": "Slack", "dockerRepository": "airbyte/source-slack", - "dockerImageTag": "0.1.9", + "dockerImageTag": "0.1.10", "documentationUrl": "https://docs.airbyte.io/integrations/sources/slack", "icon": "slack.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c7ec8009f82e..797b49dbd6c6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -349,7 +349,7 @@ - sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd name: Slack dockerRepository: airbyte/source-slack - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.io/integrations/sources/slack icon: slack.svg - sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile index 46cd52cef9d5..a116c4479830 100644 --- a/airbyte-integrations/connectors/source-slack/Dockerfile +++ b/airbyte-integrations/connectors/source-slack/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-slack diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index 2582853ac118..bb90528ff71d 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -12,13 +12,15 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/full_refresh_catalog.json" + timeout_seconds: 7200 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 7200 cursor_paths: channel_messages: ["float_ts"] threads: ["float_ts"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/full_refresh_catalog.json" + timeout_seconds: 7200 diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json b/airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json old mode 100644 new mode 100755 index 44c8b56a10a0..0b290a62aad7 --- a/airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json +++ b/airbyte-integrations/connectors/source-slack/integration_tests/full_refresh_catalog.json @@ -9,15 +9,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "channel_members", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "users", diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json old mode 100644 new mode 100755 index 2a945beab6e7..f8224f98af4e --- a/airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-slack/integration_tests/invalid_config.json @@ -1,3 +1,6 @@ { - "api_token": "wrong.api.key123" + "api_token": "invalid_api_token", + "start_date": "2022-07-22T20:00:00Z", + "lookback_window": 2, + "join_channels": true } diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-slack/integration_tests/sample_config.json new file mode 100755 index 000000000000..f85280b21048 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "api_token": "api_token", + "start_date": "2021-07-22T20:00:00Z", + "lookback_window": 2, + "join_channels": true +} diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index 58a31846f80f..81507e1f8669 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -74,16 +74,6 @@ def parse_response( json_response = response.json() yield from json_response.get(self.data_field, []) - def backoff_time(self, response: requests.Response) -> Optional[float]: - """This method is called if we run into the rate limit. - Slack puts the retry time in the `Retry-After` response header so we - we return that value. If the response is anything other than a 429 (e.g: 5XX) - fall back on default retry behavior. - - Rate Limits Docs: https://api.slack.com/docs/rate-limits#web""" - - return int(response.headers.get("Retry-After", 0)) - @property @abstractmethod def data_field(self) -> str: diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index be8971936d6f..7ab9a1ea9ef4 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -101,6 +101,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.10 | 2021-08-27 | [5697](https://github.com/airbytehq/airbyte/pull/5697) | Fixed max retries issue | | 0.1.9 | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fixed reading threads issue | | 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float_ts primary key | | 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector |