Source S3: Connector does not enforce SSL/TLS for non-S3 endpoints (#…

…17800) * Deleted ssl/tsl flag from config * Updated PR number * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Oct 12, 2022 · 5df66cd · 5df66cd
1 parent 246172b
commit 5df66cd
Show file tree

Hide file tree

Showing 8 changed files with 8 additions and 44 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -915,7 +915,7 @@
 - name: S3
   sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2
   dockerRepository: airbyte/source-s3
-  dockerImageTag: 0.1.22
+  dockerImageTag: 0.1.23
   documentationUrl: https://docs.airbyte.com/integrations/sources/s3
   icon: s3.svg
   sourceType: file

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -9429,7 +9429,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-s3:0.1.22"
+- dockerImage: "airbyte/source-s3:0.1.23"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/s3"
     changelogUrl: "https://docs.airbyte.com/integrations/sources/s3"
@@ -9707,18 +9707,6 @@
               default: ""
               order: 4
               type: "string"
-            use_ssl:
-              title: "Use TLS"
-              description: "Whether the remote server is using a secure SSL/TLS connection.\
-                \ Only relevant if using an S3-compatible, non-AWS server"
-              order: 5
-              type: "boolean"
-            verify_ssl_cert:
-              title: "Verify TLS Certificates"
-              description: "Set this to false to allow self signed certificates. Only\
-                \ relevant if using an S3-compatible, non-AWS server"
-              order: 6
-              type: "boolean"
           required:
           - "bucket"
           order: 11

diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile
@@ -17,5 +17,5 @@ COPY source_s3 ./source_s3
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.22
+LABEL io.airbyte.version=0.1.23
 LABEL io.airbyte.name=airbyte/source-s3
diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json b/airbyte-integrations/connectors/source-s3/integration_tests/config_minio.json
@@ -6,7 +6,7 @@
     "aws_access_key_id": "123456",
     "aws_secret_access_key": "123456key",
     "path_prefix": "",
-    "endpoint": "http://10.0.154.238:9000"
+    "endpoint": "http://10.0.167.14:9000"
   },
   "format": {
     "filetype": "csv"

diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json
@@ -251,18 +251,6 @@
             "default": "",
             "order": 4,
             "type": "string"
-          },
-          "use_ssl": {
-            "title": "Use TLS",
-            "description": "Whether the remote server is using a secure SSL/TLS connection. Only relevant if using an S3-compatible, non-AWS server",
-            "order": 5,
-            "type": "boolean"
-          },
-          "verify_ssl_cert": {
-            "title": "Verify TLS Certificates",
-            "description": "Set this to false to allow self signed certificates. Only relevant if using an S3-compatible, non-AWS server",
-            "order": 6,
-            "type": "boolean"
           }
         },
         "required": ["bucket"],

diff --git a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py
@@ -46,8 +46,8 @@ def _get_s3_client_args(provider: dict, config: Config) -> dict:
         # endpoint could be None or empty string, set to default Amazon endpoint in
         # this case.
         client_kv_args["endpoint_url"] = endpoint
-        client_kv_args["use_ssl"] = provider.get("use_ssl")
-        client_kv_args["verify"] = provider.get("verify_ssl_cert")
+        client_kv_args["use_ssl"] = provider.get("use_ssl", True)
+        client_kv_args["verify"] = provider.get("verify_ssl_cert", True)
 
     return client_kv_args
 

diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source.py
@@ -48,19 +48,6 @@ class Config:
         )
 
         endpoint: str = Field("", description="Endpoint to an S3 compatible service. Leave empty to use AWS.", order=4)
-        use_ssl: bool = Field(
-            default=None,
-            title="Use TLS",
-            description="Whether the remote server is using a secure SSL/TLS connection. Only relevant if using an S3-compatible, "
-            "non-AWS server",
-            order=5,
-        )
-        verify_ssl_cert: bool = Field(
-            default=None,
-            title="Verify TLS Certificates",
-            description="Set this to false to allow self signed certificates. Only relevant if using an S3-compatible, non-AWS server",
-            order=6,
-        )
 
     provider: S3Provider
 

diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md
@@ -205,7 +205,8 @@ The Jsonl parser uses pyarrow hence,only the line-delimited JSON format is suppo
 
 | Version | Date       | Pull Request                                                                                                    | Subject                                                                                 |
 | :------ | :--------- | :-------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------- |
-| 0.1.22  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304)                                                        | Migrate to per-stream state.                                                            |
+| 0.1.23  | 2022-10-10 | [17800](https://github.com/airbytehq/airbyte/pull/17800)                                                        | Deleted `use_ssl` and `verify_ssl_cert` flags and hardcoded to `True`                   |
+| 0.1.22  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304)                                                        | Migrate to per-stream state                                                             |
 | 0.1.21  | 2022-09-20 | [16921](https://github.com/airbytehq/airbyte/pull/16921)                                                        | Upgrade pyarrow                                                                         |
 | 0.1.20  | 2022-09-12 | [16607](https://github.com/airbytehq/airbyte/pull/16607)                                                        | Fix for reading jsonl files containing nested structures                                |
 | 0.1.19  | 2022-09-13 | [16631](https://github.com/airbytehq/airbyte/pull/16631)                                                        | Adjust column type to a broadest one when merging two or more json schemas              |