🐛 Source Gitlab: fix pattern field in spec file, remove unused fields…

… from config files, improve caching mechanism (#6932) * Fix SAT Fix how cache is used in this connector. Fix pattern field in spec file. Remove unused fields from invalid_config.json and sample_config.json files. * Update docs * Use cache from CDK * Bump connector's version + update docs
airbytehq · Oct 12, 2021 · 51a2e45 · 51a2e45
1 parent cb6d66d
commit 51a2e45
Show file tree

Hide file tree

Showing 8 changed files with 11 additions and 30 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json
@@ -2,6 +2,6 @@
   "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80",
   "name": "Gitlab",
   "dockerRepository": "airbyte/source-gitlab",
-  "dockerImageTag": "0.1.0",
+  "dockerImageTag": "0.1.1",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -296,7 +296,7 @@
 - sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
   name: Gitlab
   dockerRepository: airbyte/source-gitlab
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   documentationUrl: https://docs.airbyte.io/integrations/sources/gitlab
   sourceType: api
 - sourceDefinitionId: ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734

diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile
@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.version=0.1.1
 LABEL io.airbyte.name=airbyte/source-gitlab
diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json
@@ -3,8 +3,5 @@
   "private_token": "private_token-fake",
   "groups": "new-group",
   "projects": "new-ci-test",
-  "start_date": "2021-01-01T00:00:00Z",
-  "ultimate_license": false,
-  "fetch_merge_request_commits": true,
-  "fetch_pipelines_extended": true
+  "start_date": "2021-01-01T00:00:00Z"
 }
diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/sample_config.json
@@ -3,8 +3,5 @@
   "private_token": "<user`s_private_token>",
   "groups": "new-group",
   "projects": "new-ci-test",
-  "start_date": "2021-01-01T00:00:00Z",
-  "ultimate_license": false,
-  "fetch_merge_request_commits": true,
-  "fetch_pipelines_extended": true
+  "start_date": "2021-01-01T00:00:00Z"
 }
diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json
@@ -11,7 +11,7 @@
         "type": "string",
         "examples": ["gitlab.com"],
         "description": "Please enter your basic URL from Gitlab instance",
-        "pattern": ["^gitlab[a-zA-Z0-9._-]*\\.com$"]
+        "pattern": "^gitlab[a-zA-Z0-9._-]*\\.com$"
       },
       "private_token": {
         "type": "string",
@@ -34,14 +34,6 @@
         "examples": ["2021-03-01T00:00:00Z"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
       }
-    },
-    "anyOf": [
-      {
-        "required": ["groups"]
-      },
-      {
-        "required": ["projects"]
-      }
-    ]
+    }
   }
 }
diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
@@ -3,18 +3,14 @@
 #
 
 
-import tempfile
 from abc import ABC
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional
 
 import pendulum
 import requests
-import vcr
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams.http import HttpStream
 
-cache_file = tempfile.NamedTemporaryFile()
-
 
 class GitlabStream(HttpStream, ABC):
     primary_key = "id"
@@ -28,10 +24,6 @@ def __init__(self, api_url: str, **kwargs):
         super().__init__(**kwargs)
         self.api_url = api_url
 
-    def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]:
-        with vcr.use_cassette(cache_file.name, record_mode="new_episodes", serializer="json"):
-            yield from super().read_records(**kwargs)
-
     def request_params(
         self,
         stream_state: Mapping[str, Any],
@@ -153,6 +145,8 @@ def request_params(self, stream_state=None, stream_slice: Mapping[str, Any] = No
 
 
 class Groups(GitlabStream):
+    use_cache = True
+
     def __init__(self, group_ids: List, **kwargs):
         super().__init__(**kwargs)
         self.group_ids = group_ids
@@ -173,6 +167,7 @@ def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
 
 class Projects(GitlabStream):
     stream_base_params = {"statistics": 1}
+    use_cache = True
 
     def __init__(self, project_ids: List = None, **kwargs):
         super().__init__(**kwargs)
@@ -258,7 +253,6 @@ class MergeRequests(IncrementalGitlabChildStream):
 
 class MergeRequestCommits(GitlabChildStream):
     path_list = ["project_id", "iid"]
-
     path_template = "projects/{project_id}/merge_requests/{iid}"
 
     def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):

diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md
@@ -57,5 +57,6 @@ Your token should have the `read_api` scope, that Grants read access to the API,
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| 0.1.1 | 2021-10-12 | [6932](https://github.com/airbytehq/airbyte/pull/6932) | Fix pattern field in spec file, remove unused fields from config files, use cache from CDK |
 | 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | Initial Release |