🎉 Source Gitlab: add OAuth2.0 authentication support (#7506)

* added support of oauth2.0 * added OAuthFlow.java part * bumped version of the Dockerfile * corrected SAT * added unit_test for Oauth java part, added integration-test for java part * added changelog * fix java part test * updated after review * updated spec.json * updated after review * #7480 source gitlab: rm empty files * #7480 source gitlab: add single use refresh token authenticator * #7480 source gitlab: upd oauth implementation * #7480 source gitlab: refactor auth * #7480 fix oauth: java * #7480 source gitlab: review fixes * #7480 source gitlab: upd release stage * #7480 source gitlab: fix unit tests * #7480 source gitlab: update expected records * #7480 source gitlab: upd sats * #7480 source gitlab: upd expected records * #7480 source Gitlab: use SingleUseRefreshTokenOauth as is * #7480 sourcte gitlab: upd expected records * auto-bump connector version Co-authored-by: Denys Davydov <[email protected]> Co-authored-by: Augustin <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Jan 18, 2023 · b69639e · b69639e
1 parent 115144b
commit b69639e
Show file tree

Hide file tree

Showing 15 changed files with 833 additions and 291 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -605,11 +605,11 @@
 - name: Gitlab
   sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
   dockerRepository: airbyte/source-gitlab
-  dockerImageTag: 0.1.12
+  dockerImageTag: 1.0.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab
   icon: gitlab.svg
   sourceType: api
-  releaseStage: alpha
+  releaseStage: beta
 - name: Glassfrog
   sourceDefinitionId: cf8ff320-6272-4faa-89e6-4402dc17e5d5
   dockerRepository: airbyte/source-glassfrog

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -4732,31 +4732,76 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-gitlab:0.1.12"
+- dockerImage: "airbyte/source-gitlab:1.0.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/gitlab"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
-      title: "Source GitLab Singer Spec"
+      title: "Source Gitlab Spec"
       type: "object"
       required:
       - "api_url"
-      - "private_token"
       - "start_date"
+      - "credentials"
       additionalProperties: true
       properties:
-        private_token:
-          type: "string"
-          title: "Private Token"
-          description: "Log into your GitLab account and then generate a personal\
-            \ [Access Token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html)."
-          airbyte_secret: true
+        credentials:
+          title: "Authorization Method"
+          type: "object"
           order: 0
+          oneOf:
+          - type: "object"
+            title: "OAuth2.0"
+            required:
+            - "client_id"
+            - "client_secret"
+            - "refresh_token"
+            - "access_token"
+            - "token_expiry_date"
+            properties:
+              auth_type:
+                type: "string"
+                const: "oauth2.0"
+              client_id:
+                type: "string"
+                description: "The API ID of the Gitlab developer application."
+                airbyte_secret: true
+              client_secret:
+                type: "string"
+                description: "The API Secret the Gitlab developer application."
+                airbyte_secret: true
+              access_token:
+                type: "string"
+                description: "Access Token for making authenticated requests."
+                airbyte_secret: true
+              token_expiry_date:
+                type: "string"
+                description: "The date-time when the access token should be refreshed."
+                format: "date-time"
+              refresh_token:
+                type: "string"
+                description: "The key to refresh the expired access_token."
+                airbyte_secret: true
+          - title: "Private Token"
+            type: "object"
+            required:
+            - "access_token"
+            properties:
+              auth_type:
+                type: "string"
+                const: "access_token"
+              access_token:
+                type: "string"
+                title: "Private Token"
+                description: "Log into your Gitlab account and then generate a personal\
+                  \ Access Token."
+                airbyte_secret: true
         api_url:
           type: "string"
           examples:
           - "gitlab.com"
           title: "API URL"
+          default: "gitlab.com"
           description: "Please enter your basic URL from GitLab instance."
           order: 1
         start_date:
@@ -4787,6 +4832,59 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
+    advanced_auth:
+      auth_flow_type: "oauth2.0"
+      predicate_key:
+      - "credentials"
+      - "auth_type"
+      predicate_value: "oauth2.0"
+      oauth_config_specification:
+        oauth_user_input_from_connector_config_specification:
+          type: "object"
+          properties:
+            domain:
+              type: "string"
+              path_in_connector_config:
+              - "api_url"
+        complete_oauth_output_specification:
+          type: "object"
+          properties:
+            access_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "access_token"
+            refresh_token:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "refresh_token"
+            token_expiry_date:
+              type: "string"
+              format: "date-time"
+              path_in_connector_config:
+              - "credentials"
+              - "token_expiry_date"
+        complete_oauth_server_input_specification:
+          type: "object"
+          properties:
+            client_id:
+              type: "string"
+            client_secret:
+              type: "string"
+        complete_oauth_server_output_specification:
+          type: "object"
+          properties:
+            client_id:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_id"
+            client_secret:
+              type: "string"
+              path_in_connector_config:
+              - "credentials"
+              - "client_secret"
 - dockerImage: "airbyte/source-glassfrog:0.1.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/glassfrog"

diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile
@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.12
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-gitlab
diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml
@@ -4,15 +4,24 @@ acceptance_tests:
   spec:
     tests:
       - spec_path: "source_gitlab/spec.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.12"
   connection:
     tests:
       - config_path: "secrets/config.json"
         status: "succeed"
+      - config_path: "secrets/config_oauth.json"
+        status: "succeed"
       - config_path: "integration_tests/invalid_config.json"
         status: "failed"
   discovery:
     tests:
       - config_path: "secrets/config.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.12"
+      - config_path: "secrets/config_oauth.json"
+        backward_compatibility_tests_config:
+          disable_for_version: "0.1.12"
   basic_read:
     tests:
       - config_path: "secrets/config.json"
@@ -26,6 +35,9 @@ acceptance_tests:
             bypass_reason: "Group in this config does not have epics issues. This stream is tested in the above TC."
         expect_records:
           path: "integration_tests/expected_records_with_ids.jsonl"
+      - config_path: "secrets/config_oauth.json"
+        expect_records:
+          path: "integration_tests/expected_records.jsonl"
   incremental:
     tests:
       - config_path: "secrets/config_with_ids.json"

diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl