From 5df66cd572d7fcc527a5256556b34d37110e22ab Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Wed, 12 Oct 2022 16:07:22 +0200 Subject: [PATCH] Source S3: Connector does not enforce SSL/TLS for non-S3 endpoints (#17800) * Deleted ssl/tsl flag from config * Updated PR number * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 14 +------------- .../connectors/source-s3/Dockerfile | 2 +- .../source-s3/integration_tests/config_minio.json | 2 +- .../source-s3/integration_tests/spec.json | 12 ------------ .../connectors/source-s3/source_s3/s3_utils.py | 4 ++-- .../connectors/source-s3/source_s3/source.py | 13 ------------- docs/integrations/sources/s3.md | 3 ++- 8 files changed, 8 insertions(+), 44 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b67cc2a76ca7..3b6bbef558c0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -915,7 +915,7 @@ - name: S3 sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 dockerRepository: airbyte/source-s3 - dockerImageTag: 0.1.22 + dockerImageTag: 0.1.23 documentationUrl: https://docs.airbyte.com/integrations/sources/s3 icon: s3.svg sourceType: file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8287a6feea10..03caa92a3b9e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9429,7 +9429,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-s3:0.1.22" +- dockerImage: "airbyte/source-s3:0.1.23" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/s3" changelogUrl: "https://docs.airbyte.com/integrations/sources/s3" @@ -9707,18 +9707,6 @@ default: "" order: 4 type: "string" - use_ssl: - title: "Use TLS" - description: "Whether the remote server is using a secure SSL/TLS connection.\ - \ Only relevant if using an S3-compatible, non-AWS server" - order: 5 - type: "boolean" - verify_ssl_cert: - title: "Verify TLS Certificates" - description: "Set this to false to allow self signed certificates. Only\ - \ relevant if using an S3-compatible, non-AWS server" - order: 6 - type: "boolean" required: - "bucket" order: 11 diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile index 9459dc5752d5..cb908e0cd51d 100644 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ b/airbyte-integrations/connectors/source-s3/Dockerfile @@ -17,5 +17,5 @@ COPY source_s3 ./source_s3 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.22 +LABEL io.airbyte.version=0.1.23 LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json index c1ecebad5ae7..726aef0143d0 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json @@ -6,7 +6,7 @@ "aws_access_key_id": "123456", "aws_secret_access_key": "123456key", "path_prefix": "", - "endpoint": "http://10.0.154.238:9000" + "endpoint": "http://10.0.167.14:9000" }, "format": { "filetype": "csv" diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index b6c53fae4ce3..417c48058501 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -251,18 +251,6 @@ "default": "", "order": 4, "type": "string" - }, - "use_ssl": { - "title": "Use TLS", - "description": "Whether the remote server is using a secure SSL/TLS connection. Only relevant if using an S3-compatible, non-AWS server", - "order": 5, - "type": "boolean" - }, - "verify_ssl_cert": { - "title": "Verify TLS Certificates", - "description": "Set this to false to allow self signed certificates. Only relevant if using an S3-compatible, non-AWS server", - "order": 6, - "type": "boolean" } }, "required": ["bucket"], diff --git a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py index 213d0d216720..607495951fa9 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py @@ -46,8 +46,8 @@ def _get_s3_client_args(provider: dict, config: Config) -> dict: # endpoint could be None or empty string, set to default Amazon endpoint in # this case. client_kv_args["endpoint_url"] = endpoint - client_kv_args["use_ssl"] = provider.get("use_ssl") - client_kv_args["verify"] = provider.get("verify_ssl_cert") + client_kv_args["use_ssl"] = provider.get("use_ssl", True) + client_kv_args["verify"] = provider.get("verify_ssl_cert", True) return client_kv_args diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source.py index 8317f8d89710..f7b54c45f673 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source.py @@ -48,19 +48,6 @@ class Config: ) endpoint: str = Field("", description="Endpoint to an S3 compatible service. Leave empty to use AWS.", order=4) - use_ssl: bool = Field( - default=None, - title="Use TLS", - description="Whether the remote server is using a secure SSL/TLS connection. Only relevant if using an S3-compatible, " - "non-AWS server", - order=5, - ) - verify_ssl_cert: bool = Field( - default=None, - title="Verify TLS Certificates", - description="Set this to false to allow self signed certificates. Only relevant if using an S3-compatible, non-AWS server", - order=6, - ) provider: S3Provider diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 77c244fd0ef4..9ab0d44064fe 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -205,7 +205,8 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------- | -| 0.1.22 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.1.23 | 2022-10-10 | [17800](https://github.com/airbytehq/airbyte/pull/17800) | Deleted `use_ssl` and `verify_ssl_cert` flags and hardcoded to `True` | +| 0.1.22 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state | | 0.1.21 | 2022-09-20 | [16921](https://github.com/airbytehq/airbyte/pull/16921) | Upgrade pyarrow | | 0.1.20 | 2022-09-12 | [16607](https://github.com/airbytehq/airbyte/pull/16607) | Fix for reading jsonl files containing nested structures | | 0.1.19 | 2022-09-13 | [16631](https://github.com/airbytehq/airbyte/pull/16631) | Adjust column type to a broadest one when merging two or more json schemas |