diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json index 72ac8461b7d2..f10370fa5d52 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", "name": "GitHub", "dockerRepository": "airbyte/source-github", - "dockerImageTag": "0.1.7", + "dockerImageTag": "0.1.8", "documentationUrl": "https://docs.airbyte.io/integrations/sources/github", "icon": "github.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 797b49dbd6c6..ec6800bfce59 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -39,7 +39,7 @@ - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e name: GitHub dockerRepository: airbyte/source-github - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg - sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 4b8f455af5ed..c4b7a6fdfb13 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.1.8 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index 89699dd07ba1..4761883b4dec 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -17,17 +17,18 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" cursor_paths: - events: ["airbytehq/integration-test", "created_at"] comments: ["airbytehq/integration-test", "updated_at"] - pull_requests: ["airbytehq/integration-test", "updated_at"] commit_comments: ["airbytehq/integration-test", "updated_at"] - issue_milestones: ["airbytehq/integration-test", "updated_at"] commits: ["airbytehq/integration-test", "created_at"] - stargazers: ["airbytehq/integration-test", "starred_at"] - projects: ["airbytehq/integration-test", "updated_at"] - issues: ["airbytehq/integration-test", "updated_at"] + events: ["airbytehq/integration-test", "created_at"] issue_events: ["airbytehq/integration-test", "created_at"] + issue_milestones: ["airbytehq/integration-test", "updated_at"] + issues: ["airbytehq/integration-test", "updated_at"] + projects: ["airbytehq/integration-test", "updated_at"] + pull_requests: ["airbytehq/integration-test", "updated_at"] releases: ["airbytehq/integration-test", "created_at"] + review_comments: ["airbytehq/integration-test", "updated_at"] + stargazers: ["airbytehq/integration-test", "starred_at"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json index 91300c499109..dfae03d57420 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json @@ -1,17 +1,27 @@ { + "comments": { + "airbytehq/integration-test": { + "updated_at": "2121-06-30T10:22:10Z" + } + }, "commit_comments": { "airbytehq/integration-test": { "updated_at": "2121-04-30T20:36:17Z" } }, - "projects": { + "commits": { "airbytehq/integration-test": { - "updated_at": "2121-06-28T17:24:51Z" + "created_at": "2121-06-30T10:04:41Z" } }, - "releases": { + "events": { "airbytehq/integration-test": { - "created_at": "2121-06-23T23:57:07Z" + "created_at": "2121-06-29T03:44:45Z" + } + }, + "issue_events": { + "airbytehq/integration-test": { + "created_at": "2121-06-29T01:49:42Z" } }, "issue_milestones": { @@ -24,35 +34,29 @@ "updated_at": "2121-06-30T06:44:42Z" } }, - "comments": { - "airbytehq/integration-test": { - "updated_at": "2121-06-30T10:22:10Z" - } - }, - "commits": { + "projects": { "airbytehq/integration-test": { - "created_at": "2121-06-30T10:04:41Z" + "updated_at": "2121-06-28T17:24:51Z" } }, - - "stargazers": { + "pull_requests": { "airbytehq/integration-test": { - "starred_at": "2121-06-29T02:04:57Z" + "updated_at": "2121-06-28T23:36:35Z" } }, - "events": { + "releases": { "airbytehq/integration-test": { - "created_at": "2121-06-29T03:44:45Z" + "created_at": "2121-06-23T23:57:07Z" } }, - "issue_events": { + "review_comments": { "airbytehq/integration-test": { - "created_at": "2121-06-29T01:49:42Z" + "updated_at": "2121-06-23T23:57:07Z" } }, - "pull_requests": { + "stargazers": { "airbytehq/integration-test": { - "updated_at": "2121-06-28T23:36:35Z" + "starred_at": "2121-06-29T02:04:57Z" } } } diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json index 4af5492c56a2..ced72b1e4907 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json @@ -5,40 +5,37 @@ "name": "assignees", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { - "name": "reviews", + "name": "branches", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { - "name": "events", + "name": "collaborators", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], + "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { "name": "comments", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, + "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, @@ -48,10 +45,10 @@ }, { "stream": { - "name": "pull_requests", + "name": "commit_comments", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, + "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, @@ -61,36 +58,36 @@ }, { "stream": { - "name": "commit_comments", + "name": "commits", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], - "source_defined_primary_key": [["id"]] + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["sha"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["updated_at"] + "cursor_field": ["created_at"] }, { "stream": { - "name": "issue_milestones", + "name": "events", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["updated_at"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["updated_at"] + "cursor_field": ["created_at"] }, { "stream": { - "name": "commits", + "name": "issue_events", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, + "source_defined_cursor": true, "default_cursor_field": ["created_at"], "source_defined_primary_key": [["id"]] }, @@ -100,33 +97,46 @@ }, { "stream": { - "name": "collaborators", + "name": "issue_labels", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { - "name": "stargazers", + "name": "issue_milestones", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["starred_at"], - "source_defined_primary_key": [["user_id"]] + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["starred_at"] + "cursor_field": ["updated_at"] }, { "stream": { - "name": "teams", + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "organizations", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -136,7 +146,7 @@ "name": "projects", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, + "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, @@ -146,20 +156,20 @@ }, { "stream": { - "name": "issue_labels", + "name": "pull_request_stats", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { - "name": "issues", + "name": "pull_requests", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, + "source_defined_cursor": true, "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, @@ -169,10 +179,10 @@ }, { "stream": { - "name": "issue_events", + "name": "releases", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, + "source_defined_cursor": true, "default_cursor_field": ["created_at"], "source_defined_primary_key": [["id"]] }, @@ -182,16 +192,79 @@ }, { "stream": { - "name": "releases", + "name": "repositories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "review_comments", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["created_at"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "stargazers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["starred_at"], + "source_defined_primary_key": [["user_id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["starred_at"] + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json b/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json new file mode 100644 index 000000000000..a331a06cbe5c --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/branches.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "repository": { + "type": ["string"] + }, + "name": { + "type": ["null", "string"] + }, + "commit": { + "type": ["null", "object"], + "properties": { + "sha": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "protected": { + "type": ["null", "boolean"] + }, + "protection": { + "type": ["null", "object"], + "properties": { + "required_status_checks": { + "type": ["null", "object"], + "properties": { + "enforcement_level": { + "type": ["null", "string"] + }, + "contexts": { + "type": ["null", "array"], + "items": [ + { + "type": ["null", "string"] + }, + { + "type": ["null", "string"] + } + ] + } + } + } + } + }, + "protection_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json b/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json new file mode 100644 index 000000000000..d31d67cd424a --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/organizations.json @@ -0,0 +1,151 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "login": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "repos_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "hooks_url": { + "type": ["null", "string"] + }, + "issues_url": { + "type": ["null", "string"] + }, + "members_url": { + "type": ["null", "string"] + }, + "public_members_url": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "blog": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "twitter_username": { + "type": ["null", "string"] + }, + "is_verified": { + "type": ["null", "boolean"] + }, + "has_organization_projects": { + "type": ["null", "boolean"] + }, + "has_repository_projects": { + "type": ["null", "boolean"] + }, + "public_repos": { + "type": ["null", "integer"] + }, + "public_gists": { + "type": ["null", "integer"] + }, + "followers": { + "type": ["null", "integer"] + }, + "following": { + "type": ["null", "integer"] + }, + "html_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "type": { + "type": ["null", "string"] + }, + "total_private_repos": { + "type": ["null", "integer"] + }, + "owned_private_repos": { + "type": ["null", "integer"] + }, + "private_gists": { + "type": ["null", "integer"] + }, + "disk_usage": { + "type": ["null", "integer"] + }, + "collaborators": { + "type": ["null", "integer"] + }, + "billing_email": { + "type": ["null", "string"] + }, + "default_repository_permission": { + "type": ["null", "string"] + }, + "members_can_create_repositories": { + "type": ["null", "boolean"] + }, + "two_factor_requirement_enabled": { + "type": ["null", "boolean"] + }, + "members_can_create_pages": { + "type": ["null", "boolean"] + }, + "members_can_create_public_pages": { + "type": ["null", "boolean"] + }, + "members_can_create_private_pages": { + "type": ["null", "boolean"] + }, + "plan": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "space": { + "type": ["null", "integer"] + }, + "private_repos": { + "type": ["null", "integer"] + }, + "filled_seats": { + "type": ["null", "integer"] + }, + "seats": { + "type": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_stats.json b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_stats.json new file mode 100644 index 000000000000..3b221876cfae --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_request_stats.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "repository": { + "type": ["string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "merged": { + "type": ["null", "boolean"] + }, + "mergeable": { + "type": ["null", "boolean"] + }, + "rebaseable": { + "type": ["null", "boolean"] + }, + "mergeable_state": { + "type": ["null", "string"] + }, + "merged_by": { + "$ref": "user.json" + }, + "comments": { + "type": ["null", "integer"] + }, + "review_comments": { + "type": ["null", "integer"] + }, + "maintainer_can_modify": { + "type": ["null", "boolean"] + }, + "commits": { + "type": ["null", "integer"] + }, + "additions": { + "type": ["null", "integer"] + }, + "deletions": { + "type": ["null", "integer"] + }, + "changed_files": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json b/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json new file mode 100644 index 000000000000..97ff519de8e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/repositories.json @@ -0,0 +1,265 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "full_name": { + "type": ["null", "string"] + }, + "owner": { + "$ref": "user.json" + }, + "private": { + "type": ["null", "boolean"] + }, + "html_url": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "fork": { + "type": ["null", "boolean"] + }, + "url": { + "type": ["null", "string"] + }, + "archive_url": { + "type": ["null", "string"] + }, + "assignees_url": { + "type": ["null", "string"] + }, + "blobs_url": { + "type": ["null", "string"] + }, + "branches_url": { + "type": ["null", "string"] + }, + "collaborators_url": { + "type": ["null", "string"] + }, + "comments_url": { + "type": ["null", "string"] + }, + "commits_url": { + "type": ["null", "string"] + }, + "compare_url": { + "type": ["null", "string"] + }, + "contents_url": { + "type": ["null", "string"] + }, + "contributors_url": { + "type": ["null", "string"] + }, + "deployments_url": { + "type": ["null", "string"] + }, + "downloads_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "forks_url": { + "type": ["null", "string"] + }, + "git_commits_url": { + "type": ["null", "string"] + }, + "git_refs_url": { + "type": ["null", "string"] + }, + "git_tags_url": { + "type": ["null", "string"] + }, + "git_url": { + "type": ["null", "string"] + }, + "issue_comment_url": { + "type": ["null", "string"] + }, + "issue_events_url": { + "type": ["null", "string"] + }, + "issues_url": { + "type": ["null", "string"] + }, + "keys_url": { + "type": ["null", "string"] + }, + "labels_url": { + "type": ["null", "string"] + }, + "languages_url": { + "type": ["null", "string"] + }, + "merges_url": { + "type": ["null", "string"] + }, + "milestones_url": { + "type": ["null", "string"] + }, + "notifications_url": { + "type": ["null", "string"] + }, + "pulls_url": { + "type": ["null", "string"] + }, + "releases_url": { + "type": ["null", "string"] + }, + "ssh_url": { + "type": ["null", "string"] + }, + "stargazers_url": { + "type": ["null", "string"] + }, + "statuses_url": { + "type": ["null", "string"] + }, + "subscribers_url": { + "type": ["null", "string"] + }, + "subscription_url": { + "type": ["null", "string"] + }, + "tags_url": { + "type": ["null", "string"] + }, + "teams_url": { + "type": ["null", "string"] + }, + "trees_url": { + "type": ["null", "string"] + }, + "clone_url": { + "type": ["null", "string"] + }, + "mirror_url": { + "type": ["null", "string"] + }, + "hooks_url": { + "type": ["null", "string"] + }, + "svn_url": { + "type": ["null", "string"] + }, + "homepage": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + }, + "forks_count": { + "type": ["null", "integer"] + }, + "stargazers_count": { + "type": ["null", "integer"] + }, + "watchers_count": { + "type": ["null", "integer"] + }, + "size": { + "type": ["null", "integer"] + }, + "default_branch": { + "type": ["null", "string"] + }, + "open_issues_count": { + "type": ["null", "integer"] + }, + "is_template": { + "type": ["null", "boolean"] + }, + "topics": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "license": { + "type": ["null", "object"], + "properties": { + "key": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "spdx_id": { + "type": ["null", "string"] + }, + "node_id": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "has_issues": { + "type": ["null", "boolean"] + }, + "has_projects": { + "type": ["null", "boolean"] + }, + "has_wiki": { + "type": ["null", "boolean"] + }, + "has_pages": { + "type": ["null", "boolean"] + }, + "has_downloads": { + "type": ["null", "boolean"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "disabled": { + "type": ["null", "boolean"] + }, + "visibility": { + "type": ["null", "string"] + }, + "pushed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "permissions": { + "type": ["null", "object"], + "properties": { + "admin": { + "type": ["null", "boolean"] + }, + "push": { + "type": ["null", "boolean"] + }, + "pull": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/review_comments.json b/airbyte-integrations/connectors/source-github/source_github/schemas/review_comments.json new file mode 100644 index 000000000000..4506f45bd383 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/review_comments.json @@ -0,0 +1,112 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "repository": { + "type": ["string"] + }, + "url": { + "type": ["null", "string"] + }, + "pull_request_review_id": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "diff_hunk": { + "type": ["null", "string"] + }, + "path": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "original_position": { + "type": ["null", "integer"] + }, + "commit_id": { + "type": ["null", "string"] + }, + "original_commit_id": { + "type": ["null", "string"] + }, + "in_reply_to_id": { + "type": ["null", "integer"] + }, + "user": { + "$ref": "user.json" + }, + "body": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "html_url": { + "type": ["null", "string"] + }, + "pull_request_url": { + "type": ["null", "string"] + }, + "author_association": { + "type": ["null", "string"] + }, + "_links": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "html": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + }, + "pull_request": { + "type": ["null", "object"], + "properties": { + "href": { + "type": ["null", "string"] + } + } + } + } + }, + "start_line": { + "type": ["null", "integer"] + }, + "original_start_line": { + "type": ["null", "integer"] + }, + "start_side": { + "type": ["null", "string"] + }, + "line": { + "type": ["null", "integer"] + }, + "original_line": { + "type": ["null", "integer"] + }, + "side": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/tags.json b/airbyte-integrations/connectors/source-github/source_github/schemas/tags.json new file mode 100644 index 000000000000..bc7d9feb15b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/tags.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "repository": { + "type": ["string"] + }, + "name": { + "type": ["null", "string"] + }, + "commit": { + "type": ["null", "object"], + "properties": { + "sha": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "zipball_url": { + "type": ["null", "string"] + }, + "tarball_url": { + "type": ["null", "string"] + }, + "node_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json b/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json index 4ae3f2f88fb1..9cdab14307cd 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/teams.json @@ -2,8 +2,11 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "organization": { + "type": ["null", "string"] + }, "repository": { - "type": ["string"] + "type": ["null", "string"] }, "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/users.json b/airbyte-integrations/connectors/source-github/source_github/schemas/users.json new file mode 100644 index 000000000000..7ae4d47b16c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/users.json @@ -0,0 +1,63 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "organization": { + "type": ["null", "string"] + }, + "login": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "gravatar_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "followers_url": { + "type": ["null", "string"] + }, + "following_url": { + "type": ["null", "string"] + }, + "gists_url": { + "type": ["null", "string"] + }, + "starred_url": { + "type": ["null", "string"] + }, + "subscriptions_url": { + "type": ["null", "string"] + }, + "organizations_url": { + "type": ["null", "string"] + }, + "repos_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "received_events_url": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "site_admin": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index eb4c993b5f1f..922e4cb27925 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -34,6 +34,7 @@ from .streams import ( Assignees, + Branches, Collaborators, Comments, CommitComments, @@ -43,13 +44,18 @@ IssueLabels, IssueMilestones, Issues, + Organizations, Projects, PullRequests, + PullRequestStats, Releases, Repositories, + ReviewComments, Reviews, Stargazers, + Tags, Teams, + Users, ) TOKEN_SEPARATOR = "," @@ -98,11 +104,14 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = self._get_authenticator(config["access_token"]) repositories = self._generate_repositories(config=config, authenticator=authenticator) + organizations = list({org.split("/")[0] for org in repositories}) full_refresh_args = {"authenticator": authenticator, "repositories": repositories} incremental_args = {**full_refresh_args, "start_date": config["start_date"]} + organization_args = {"authenticator": authenticator, "organizations": organizations} return [ Assignees(**full_refresh_args), + Branches(**full_refresh_args), Collaborators(**full_refresh_args), Comments(**incremental_args), CommitComments(**incremental_args), @@ -112,10 +121,16 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: IssueLabels(**full_refresh_args), IssueMilestones(**incremental_args), Issues(**incremental_args), + Organizations(**organization_args), Projects(**incremental_args), + PullRequestStats(**full_refresh_args), PullRequests(**incremental_args), Releases(**incremental_args), + Repositories(**organization_args), + ReviewComments(**incremental_args), Reviews(**full_refresh_args), Stargazers(**incremental_args), - Teams(**full_refresh_args), + Tags(**full_refresh_args), + Teams(**organization_args), + Users(**organization_args), ] diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 0f1e42032415..9885022ecea4 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -164,7 +164,7 @@ def parse_response( for record in response.json(): # GitHub puts records in an array. yield self.transform(record=record, repository=stream_slice["repository"]) - def transform(self, record: MutableMapping[str, Any], repository: str) -> MutableMapping[str, Any]: + def transform(self, record: MutableMapping[str, Any], repository: str = None, organization: str = None) -> MutableMapping[str, Any]: """ Use this method to: - remove excessive fields from record; @@ -193,7 +193,10 @@ def transform(self, record: MutableMapping[str, Any], repository: str) -> Mutabl record[f"{field}_id"] = field_value.get("id") if field_value else None elif isinstance(field_value, list): record[field] = [value.get("id") for value in field_value] - record["repository"] = repository + if repository: + record["repository"] = repository + if organization: + record["organization"] = organization return record @@ -272,34 +275,41 @@ def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[ return params -class Repositories(GithubStream): +# Below are full refresh streams + + +class Assignees(GithubStream): """ - This stream is technical and not intended for the user, it is used only to obtain repositories for organizations. - API docs: https://docs.github.com/en/rest/reference/repos#list-organization-repositories + API docs: https://docs.github.com/en/rest/reference/issues#list-assignees """ - def __init__(self, organizations: List[str], **kwargs): - super(GithubStream, self).__init__(**kwargs) - self.organizations = organizations - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for organization in self.organizations: - yield {"organization": organization} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"orgs/{stream_slice['organization']}/repos" +class PullRequestStats(GithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/pulls#get-a-pull-request + """ - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() + @property + def record_keys(self) -> List[str]: + return list(self.get_json_schema()["properties"].keys()) + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"repos/{stream_slice['repository']}/pulls/{stream_slice['pull_request_number']}" -# Below are full refresh streams + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for stream_slice in super().stream_slices(**kwargs): + pull_requests_stream = PullRequests(authenticator=self.authenticator, repositories=[stream_slice["repository"]], start_date="") + for pull_request in pull_requests_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): + yield {"pull_request_number": pull_request["number"], "repository": stream_slice["repository"]} + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + yield self.transform(response.json(), repository=stream_slice["repository"]) -class Assignees(GithubStream): - """ - API docs: https://docs.github.com/en/rest/reference/issues#list-assignees - """ + def transform(self, record: MutableMapping[str, Any], repository: str = None) -> MutableMapping[str, Any]: + record = super().transform(record=record, repository=repository) + return {key: value for key, value in record.items() if key in self.record_keys} class Reviews(GithubStream): @@ -319,6 +329,17 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield {"pull_request_number": pull_request["number"], "repository": stream_slice["repository"]} +class Branches(GithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/repos#list-branches + """ + + primary_key = None + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"repos/{stream_slice['repository']}/branches" + + class Collaborators(GithubStream): """ API docs: https://docs.github.com/en/rest/reference/repos#list-repository-collaborators @@ -334,14 +355,74 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/labels" -class Teams(GithubStream): +class Organizations(GithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/orgs#get-an-organization + """ + + def __init__(self, organizations: List[str], **kwargs): + super(GithubStream, self).__init__(**kwargs) + self.organizations = organizations + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + for organization in self.organizations: + yield {"organization": organization} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"orgs/{stream_slice['organization']}" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield response.json() + + +class Repositories(Organizations): + """ + API docs: https://docs.github.com/en/rest/reference/repos#list-organization-repositories + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"orgs/{stream_slice['organization']}/repos" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + for record in response.json(): # GitHub puts records in an array. + yield self.transform(record=record, organization=stream_slice["organization"]) + + +class Tags(GithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/repos#list-repository-tags + """ + + primary_key = None + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"repos/{stream_slice['repository']}/tags" + + +class Teams(Organizations): """ API docs: https://docs.github.com/en/rest/reference/teams#list-teams """ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - owner, _ = stream_slice["repository"].split("/") - return f"orgs/{owner}/teams" + return f"orgs/{stream_slice['organization']}/teams" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + for record in response.json(): + yield self.transform(record=record, organization=stream_slice["organization"]) + + +class Users(Organizations): + """ + API docs: https://docs.github.com/en/rest/reference/orgs#list-organization-members + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"orgs/{stream_slice['organization']}/members" + + def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + for record in response.json(): + yield self.transform(record, organization=stream_slice["organization"]) # Below are semi incremental streams @@ -355,7 +436,7 @@ class Releases(SemiIncrementalGithubStream): cursor_field = "created_at" fields_to_minimize = ("author",) - def transform(self, record: MutableMapping[str, Any], repository: str = None) -> MutableMapping[str, Any]: + def transform(self, record: MutableMapping[str, Any], repository: str = None, **kwargs) -> MutableMapping[str, Any]: record = super().transform(record=record, repository=repository) assets = record.get("assets", []) @@ -409,7 +490,7 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/pulls" - def transform(self, record: MutableMapping[str, Any], repository: str) -> MutableMapping[str, Any]: + def transform(self, record: MutableMapping[str, Any], repository: str = None, **kwargs) -> MutableMapping[str, Any]: record = super().transform(record=record, repository=repository) for nested in ("head", "base"): @@ -476,7 +557,7 @@ def request_headers(self, **kwargs) -> Mapping[str, Any]: return {**base_headers, **headers} - def transform(self, record: MutableMapping[str, Any], repository: str) -> MutableMapping[str, Any]: + def transform(self, record: MutableMapping[str, Any], repository: str = None, **kwargs) -> MutableMapping[str, Any]: """ We need to provide the "user_id" for the primary_key attribute and don't remove the whole "user" block from the record. @@ -546,7 +627,7 @@ class Commits(IncrementalGithubStream): "committer", ) - def transform(self, record: MutableMapping[str, Any], repository: str = None) -> MutableMapping[str, Any]: + def transform(self, record: MutableMapping[str, Any], repository: str = None, **kwargs) -> MutableMapping[str, Any]: record = super().transform(record=record, repository=repository) # Record of the `commits` stream doesn't have an updated_at/created_at field at the top level (so we could @@ -576,3 +657,14 @@ class Issues(IncrementalGithubStream): "sort": "updated", "direction": "asc", } + + +class ReviewComments(IncrementalGithubStream): + """ + API docs: https://docs.github.com/en/rest/reference/pulls#list-review-comments-in-a-repository + """ + + page_size = 30 # `review-comments` is a large stream so it's better to set smaller page size. + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"repos/{stream_slice['repository']}/pulls/comments" diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index 82857bef9538..7c6d5259bd6a 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -9,23 +9,30 @@ The GitHub source supports both Full Refresh and Incremental syncs. You can choo This connector outputs the following full refresh streams: * [Assignees](https://docs.github.com/en/rest/reference/issues#list-assignees) -* [Reviews](https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request) +* [Branches](https://docs.github.com/en/rest/reference/repos#list-branches) * [Collaborators](https://docs.github.com/en/rest/reference/repos#list-repository-collaborators) -* [Teams](https://docs.github.com/en/rest/reference/teams#list-teams) * [Issue labels](https://docs.github.com/en/free-pro-team@latest/rest/reference/issues#list-labels-for-a-repository) +* [Organizations](https://docs.github.com/en/rest/reference/orgs#get-an-organization) +* [Pull Request Stats](https://docs.github.com/en/rest/reference/pulls#get-a-pull-request) +* [Repositories](https://docs.github.com/en/rest/reference/repos#list-organization-repositories) +* [Reviews](https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request) +* [Tags](https://docs.github.com/en/rest/reference/repos#list-repository-tags) +* [Teams](https://docs.github.com/en/rest/reference/teams#list-teams) +* [Users](https://docs.github.com/en/rest/reference/orgs#list-organization-members) This connector outputs the following incremental streams: * [Comments](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository) * [Commits](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository) -* [Issues](https://docs.github.com/en/rest/reference/issues#list-repository-issues) * [Commit comments](https://docs.github.com/en/rest/reference/repos#list-commit-comments-for-a-repository) * [Events](https://docs.github.com/en/rest/reference/activity#list-repository-events) +* [Issues](https://docs.github.com/en/rest/reference/issues#list-repository-issues) * [Issue events](https://docs.github.com/en/rest/reference/issues#list-issue-events-for-a-repository) * [Issue milestones](https://docs.github.com/en/rest/reference/issues#list-milestones) * [Projects](https://docs.github.com/en/rest/reference/projects#list-repository-projects) * [Pull requests](https://docs.github.com/en/rest/reference/pulls#list-pull-requests) * [Releases](https://docs.github.com/en/rest/reference/repos#list-releases) +* [Review Comments](https://docs.github.com/en/rest/reference/pulls#list-review-comments-in-a-repository) * [Stargazers](https://docs.github.com/en/rest/reference/activity#list-stargazers) ### Notes @@ -86,6 +93,7 @@ Your token should have at least the `repo` scope. Depending on which streams you | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.8 | 2021-09-01 | [5757](https://github.com/airbytehq/airbyte/pull/5757) | Add more streams | | 0.1.7 | 2021-08-27 | [5696](https://github.com/airbytehq/airbyte/pull/5696) | Handle negative backoff values | | 0.1.6 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5223) | Add MultipleTokenAuthenticator | | 0.1.5 | 2021-08-18 | [5456](https://github.com/airbytehq/airbyte/pull/5456) | Fix set up validation |