Source Gitlab: add GroupIssueBoards stream (#13252)

* GitLab Source: add GroupIssueBoards stream * Address stream schema comments * Address comments * Bump version * Add as empty stream
airbytehq · Jun 28, 2022 · 06d588b · 06d588b
1 parent f5843b0
commit 06d588b
Show file tree

Hide file tree

Showing 8 changed files with 82 additions and 2 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -311,7 +311,7 @@
 - name: Gitlab
   sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
   dockerRepository: airbyte/source-gitlab
-  dockerImageTag: 0.1.5
+  dockerImageTag: 0.1.6
   documentationUrl: https://docs.airbyte.io/integrations/sources/gitlab
   icon: gitlab.svg
   sourceType: api

diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile
@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.5
+LABEL io.airbyte.version=0.1.6
 LABEL io.airbyte.name=airbyte/source-gitlab
diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml
@@ -12,6 +12,7 @@ tests:
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams: ["group_issue_boards"]
 # We cannot use these tests for testing Incremental, since for Gitlab the State is saved for each Project separately,
 # and the Acceptance Tests at this stage do not support this functionality.
 # Therefore, we hardcode the cursor_paths for our config.

diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json
@@ -43,6 +43,16 @@
       "destination_sync_mode": "append",
       "cursor_field": ["created_at"]
     },
+    {
+      "stream": {
+        "name": "group_issue_boards",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "issues",

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_issue_boards.json
@@ -0,0 +1,61 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "name": {
+      "type": ["null", "string"]
+    },
+    "hide_backlog_list": {
+      "type": ["null", "boolean"]
+    },
+    "hide_closed_list": {
+      "type": ["null", "boolean"]
+    },
+    "project": {
+      "type": ["null", "integer"]
+    },
+    "lists": {
+      "type": ["null", "array"],
+      "items": {
+        "type": ["null", "object"],
+        "properties": {
+          "id": {
+            "type": ["null", "integer"]
+          },
+          "label": {
+            "type": ["null", "object"],
+            "properties": {
+              "id": {
+                "type": ["null", "integer"]
+              },
+              "name": {
+                "type": ["null", "string"]
+              },
+              "description": {
+                "type": ["null", "string"]
+              },
+              "description_html": {
+                "type": ["null", "string"]
+              },
+              "text_color": {
+                "type": ["null", "string"]
+              },
+              "color": {
+                "type": ["null", "string"]
+              }
+            }
+          },
+          "position": {
+            "type": ["null", "integer"]
+          }
+        }
+      }
+    },
+    "group_id": {
+      "type": ["null", "integer"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py
@@ -17,6 +17,7 @@
     EpicIssues,
     Epics,
     GitlabStream,
+    GroupIssueBoards,
     GroupLabels,
     GroupMembers,
     GroupMilestones,
@@ -105,6 +106,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             Commits(parent_stream=projects, repository_part=True, start_date=config["start_date"], **auth_params),
             epics,
             EpicIssues(parent_stream=epics, **auth_params),
+            GroupIssueBoards(parent_stream=groups, **auth_params),
             Issues(parent_stream=projects, start_date=config["start_date"], **auth_params),
             Jobs(parent_stream=pipelines, **auth_params),
             ProjectMilestones(parent_stream=projects, **auth_params),

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
@@ -306,6 +306,11 @@ def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
         return record
 
 
+class GroupIssueBoards(GitlabChildStream):
+    path_template = "groups/{id}/boards"
+    flatten_parent_id = True
+
+
 class Users(GitlabChildStream):
     pass
 

diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md
@@ -63,6 +63,7 @@ GitLab source is working with GitLab API v4. It can also work with self-hosted G
 
 | Version | Date       | Pull Request                                             | Subject |
 |:--------|:-----------|:---------------------------------------------------------| :--- |
+| 0.1.6   | 2022-06-23 | [13252](https://github.com/airbytehq/airbyte/pull/13252) | Add GroupIssueBoards stream |
 | 0.1.5   | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` |
 | 0.1.4   | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config |
 | 0.1.3   | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991)   | Update connector fields title/description |