diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6c8ad205e653..54c7ef79e6b5 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.29.17-alpha +current_version = 0.29.21-alpha commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index 9e2e0c8d9b5c..0c428e9253fa 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -VERSION=0.29.17-alpha +VERSION=0.29.21-alpha # Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_USER=docker diff --git a/.github/ISSUE_TEMPLATE/new-integration-request.md b/.github/ISSUE_TEMPLATE/new-integration-request.md index 5a07428910ae..676b89d0899f 100644 --- a/.github/ISSUE_TEMPLATE/new-integration-request.md +++ b/.github/ISSUE_TEMPLATE/new-integration-request.md @@ -12,9 +12,10 @@ assignees: '' * Do you need a specific version of the underlying data source e.g: you specifically need support for an older version of the API or DB? ## Describe the context around this new connector -* Which team in your company wants this integration, what for? This helps us understand the use case. +* Why do you need this integration? How does your team intend to use the data? This helps us understand the use case. * How often do you want to run syncs? * If this is an API source connector, which entities/endpoints do you need supported? +* If the connector is for a paid service, can we name you as a mutual user when we subscribe for an account? Which company should we name? ## Describe the alternative you are considering or using What are you considering doing if you don’t have this integration through Airbyte? diff --git a/.github/labeler.yml b/.github/labeler.yml index 9a13f4469693..4c03b5d9929e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -36,3 +36,9 @@ CDK: normalization: - airbyte-integrations/bases/base-normalization/* - airbyte-integrations/bases/base-normalization/**/* + +kubernetes: + - kube/* + - kube/**/* + - charts/* + - charts/**/* diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 0a331e0dcdf3..b810bef4e8e2 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -415,6 +415,7 @@ jobs: # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest. needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner + environment: more-secrets name: Acceptance Tests (Kube) timeout-minutes: 90 steps: @@ -483,11 +484,25 @@ jobs: df -h docker system df + - name: Run GCP End-to-End Acceptance Tests + env: + USER: root + HOME: /home/runner + AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} + SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} + SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }} + run: | + CI=true ./tools/bin/gcp_acceptance_tests.sh + - name: Run Kubernetes End-to-End Acceptance Tests env: USER: root HOME: /home/runner AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }} + SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }} + SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }} + SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }} run: | CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh # In case of self-hosted EC2 errors, remove this block. diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml new file mode 100644 index 000000000000..22fae932c706 --- /dev/null +++ b/.github/workflows/helm.yaml @@ -0,0 +1,74 @@ +name: Helm +on: + push: + paths: + - ".github/workflows/helm.yaml" + - "charts/**" + pull_request: + paths: + - ".github/workflows/helm.yaml" + - "charts/**" +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - name: Setup Kubectl + uses: azure/setup-kubectl@v1 + - name: Setup Helm + uses: azure/setup-helm@v1 + with: + version: "3.6.3" + - name: Lint Chart + working-directory: ./charts/airbyte + run: ./ci.sh lint + + generate-docs: + name: Generate Docs Parameters + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v2 + - name: Checkout bitnami-labs/readme-generator-for-helm + uses: actions/checkout@v2 + with: + repository: "bitnami-labs/readme-generator-for-helm" + ref: "55cab5dd2191c4ffa7245cfefa428d4d9bb12730" + path: readme-generator-for-helm + - name: Install readme-generator-for-helm dependencies + working-directory: readme-generator-for-helm + run: npm install -g + - name: Test can update README with generated parameters + working-directory: charts/airbyte + run: ./ci.sh check-docs-updated + + install: + name: Install + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - name: Setup Kubectl + uses: azure/setup-kubectl@v1 + - name: Setup Helm + uses: azure/setup-helm@v1 + with: + version: "3.6.3" + - name: Setup Kind Cluster + uses: helm/kind-action@v1.2.0 + with: + version: "v0.11.1" + image: "kindest/node:v1.21.1" + - name: Install airbyte chart + working-directory: ./charts/airbyte + run: ./ci.sh install + - if: always() + name: Print diagnostics + working-directory: ./charts/airbyte + run: ./ci.sh diagnostics + - if: success() + name: Test airbyte chart + working-directory: ./charts/airbyte + run: ./ci.sh test diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index e21fde645be2..7c584b174565 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -117,6 +117,7 @@ jobs: GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }} GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }} + GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }} HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }} HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }} @@ -124,6 +125,7 @@ jobs: ITERABLE_INTEGRATION_TEST_CREDS: ${{ secrets.ITERABLE_INTEGRATION_TEST_CREDS }} JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} + LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }} LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }} MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} @@ -132,6 +134,8 @@ jobs: PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }} POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }} + MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }} + MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} @@ -179,6 +183,8 @@ jobs: SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }} SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }} DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} + MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }} + MSSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MSSQL_SSH_PWD_TEST_CREDS }} - run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index d32505fd0bbe..824601b6b5fe 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -112,6 +112,7 @@ jobs: GOOGLE_SHEETS_TESTS_CREDS: ${{ secrets.GOOGLE_SHEETS_TESTS_CREDS }} GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS: ${{ secrets.GOOGLE_WORKSPACE_ADMIN_REPORTS_TEST_CREDS }} GREENHOUSE_TEST_CREDS: ${{ secrets.GREENHOUSE_TEST_CREDS }} + GREENHOUSE_TEST_CREDS_LIMITED: ${{ secrets.GREENHOUSE_TEST_CREDS_LIMITED }} HARVEST_INTEGRATION_TESTS_CREDS: ${{ secrets.HARVEST_INTEGRATION_TESTS_CREDS }} HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} INSTAGRAM_INTEGRATION_TESTS_CREDS: ${{ secrets.INSTAGRAM_INTEGRATION_TESTS_CREDS }} @@ -120,6 +121,7 @@ jobs: JIRA_INTEGRATION_TEST_CREDS: ${{ secrets.JIRA_INTEGRATION_TEST_CREDS }} KLAVIYO_TEST_CREDS: ${{ secrets.KLAVIYO_TEST_CREDS }} SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }} + LEVER_HIRING_INTEGRATION_TEST_CREDS: ${{ secrets.LEVER_HIRING_INTEGRATION_TEST_CREDS }} LOOKER_INTEGRATION_TEST_CREDS: ${{ secrets.LOOKER_INTEGRATION_TEST_CREDS }} MAILCHIMP_TEST_CREDS: ${{ secrets.MAILCHIMP_TEST_CREDS }} MICROSOFT_TEAMS_TEST_CREDS: ${{ secrets.MICROSOFT_TEAMS_TEST_CREDS }} @@ -128,6 +130,8 @@ jobs: PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTGRES_SSH_KEY_TEST_CREDS: ${{ secrets.POSTGRES_SSH_KEY_TEST_CREDS }} POSTGRES_SSH_PWD_TEST_CREDS: ${{ secrets.POSTGRES_SSH_PWD_TEST_CREDS }} + MYSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MYSQL_SSH_KEY_TEST_CREDS }} + MYSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MYSQL_SSH_PWD_TEST_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} @@ -174,6 +178,8 @@ jobs: SOURCE_BAMBOO_HR_CREDS: ${{ secrets.SOURCE_BAMBOO_HR_CREDS }} SOURCE_BIGCOMMERCE_CREDS: ${{ secrets.SOURCE_BIGCOMMERCE_CREDS }} DESTINATION_DATABRICKS_CREDS: ${{ secrets.DESTINATION_DATABRICKS_CREDS }} + MSSQL_SSH_KEY_TEST_CREDS: ${{ secrets.MSSQL_SSH_KEY_TEST_CREDS }} + MSSQL_SSH_PWD_TEST_CREDS: ${{ secrets.MSSQL_SSH_PWD_TEST_CREDS }} - run: | ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} name: test ${{ github.event.inputs.connector }} diff --git a/.gitignore b/.gitignore index 42c590666322..f02f5b77b68b 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,21 @@ __pycache__ .ipynb_checkpoints .pytest_ +# Python unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + # dbt profiles.yml diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 583a14909088..c1e94241c04a 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1234,7 +1234,7 @@ paths: post: tags: - oauth - summary: Given a source def ID and optional workspaceID generate an access/refresh token etc. + summary: Given a source def ID generate an access/refresh token etc. operationId: completeSourceOAuth requestBody: content: @@ -1280,7 +1280,7 @@ paths: post: tags: - oauth - summary: + summary: Given a destination def ID generate an access/refresh token etc. operationId: completeDestinationOAuth requestBody: content: @@ -1410,6 +1410,29 @@ paths: $ref: "#/components/schemas/WebBackendConnectionRead" "422": $ref: "#/components/responses/InvalidInputResponse" + /v1/web_backend/connections/search: + post: + tags: + - web_backend + summary: Search connections + operationId: webBackendConnectionSearch + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/WebBackendConnectionSearch" + required: true + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/WebBackendConnectionReadList" + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" /v1/web_backend/sources/create: post: tags: @@ -1700,6 +1723,8 @@ components: type: array items: $ref: "#/components/schemas/Notification" + displaySetupWizard: + type: boolean Notification: type: object required: @@ -1906,6 +1931,34 @@ components: description: The specification for what values are required to configure the sourceDefinition. type: object example: { user: { type: string } } + SourceAuthSpecification: + $ref: "#/components/schemas/AuthSpecification" + AuthSpecification: + type: object + properties: + auth_type: + type: string + enum: ["oauth2.0"] # Future auth types should be added here + oauth2Specification: + "$ref": "#/components/schemas/OAuth2Specification" + OAuth2Specification: + description: An object containing any metadata needed to describe this connector's Oauth flow + type: object + properties: + oauthFlowInitParameters: + description: + "Pointers to the fields in the ConnectorSpecification which are needed to obtain the initial refresh/access tokens for the OAuth flow. + Each inner array represents the path in the ConnectorSpecification of the referenced field. + For example. + Assume the ConnectorSpecification contains params 'app_secret', 'app_id' which are needed to get the initial refresh token. + If they are not nested in the config, then the array would look like this [['app_secret'], ['app_id']] + If they are nested inside, say, an object called 'auth_params' then this array would be [['auth_params', 'app_secret'], ['auth_params', 'app_id']]" + type: array + items: + description: A list of strings which describes each parameter's path inside the ConnectionSpecification + type: array + items: + type: string SourceDefinitionSpecificationRead: type: object required: @@ -1918,6 +1971,8 @@ components: type: string connectionSpecification: $ref: "#/components/schemas/SourceDefinitionSpecification" + authSpecification: + $ref: "#/components/schemas/SourceAuthSpecification" jobInfo: $ref: "#/components/schemas/SynchronousJobRead" # SOURCE @@ -2014,10 +2069,27 @@ components: $ref: "#/components/schemas/AirbyteCatalog" jobInfo: $ref: "#/components/schemas/SynchronousJobRead" + SourceSearch: + type: object + properties: + sourceDefinitionId: + $ref: "#/components/schemas/SourceDefinitionId" + sourceId: + $ref: "#/components/schemas/SourceId" + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + connectionConfiguration: + $ref: "#/components/schemas/SourceConfiguration" + name: + type: string + sourceName: + type: string # DESTINATION DEFINITION DestinationDefinitionId: type: string format: uuid + DestinationAuthSpecification: + $ref: "#/components/schemas/AuthSpecification" DestinationDefinitionIdRequestBody: type: object required: @@ -2101,6 +2173,8 @@ components: type: string connectionSpecification: $ref: "#/components/schemas/DestinationDefinitionSpecification" + authSpecification: + $ref: "#/components/schemas/DestinationAuthSpecification" jobInfo: $ref: "#/components/schemas/SynchronousJobRead" supportedDestinationSyncModes: @@ -2196,6 +2270,21 @@ components: type: array items: $ref: "#/components/schemas/DestinationRead" + DestinationSearch: + type: object + properties: + destinationDefinitionId: + $ref: "#/components/schemas/DestinationDefinitionId" + destinationId: + $ref: "#/components/schemas/DestinationId" + workspaceId: + $ref: "#/components/schemas/WorkspaceId" + connectionConfiguration: + $ref: "#/components/schemas/DestinationConfiguration" + name: + type: string + destinationName: + type: string # CONNECTION ConnectionId: type: string @@ -2408,6 +2497,35 @@ components: $ref: "#/components/schemas/ConnectionStatus" resourceRequirements: $ref: "#/components/schemas/ResourceRequirements" + WebBackendConnectionSearch: + type: object + properties: + connectionId: + $ref: "#/components/schemas/ConnectionId" + name: + type: string + namespaceDefinition: + $ref: "#/components/schemas/NamespaceDefinitionType" + namespaceFormat: + type: string + description: Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'. + default: null + example: "${SOURCE_NAMESPACE}" + prefix: + type: string + description: Prefix that will be prepended to the name of each stream when it is written to the destination. + sourceId: + $ref: "#/components/schemas/SourceId" + destinationId: + $ref: "#/components/schemas/DestinationId" + schedule: + $ref: "#/components/schemas/ConnectionSchedule" + status: + $ref: "#/components/schemas/ConnectionStatus" + source: + $ref: "#/components/schemas/SourceSearch" + destination: + $ref: "#/components/schemas/DestinationSearch" ConnectionReadList: type: object required: @@ -3062,7 +3180,7 @@ components: SetInstancewideSourceOauthParamsRequestBody: type: object required: - - sourceConnectorDefinitionId + - sourceDefinitionId - params properties: sourceDefinitionId: diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index ab06ebaa2680..e83176b4172a 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.1.23 +Added the ability to use caching for efficient synchronization of nested streams. + +## 0.1.22 +Allow passing custom headers to request in `OAuth2Authenticator.refresh_access_token()`: https://github.com/airbytehq/airbyte/pull/6219 + +## 0.1.21 +Resolve nested schema references and move external references to single schema definitions. + +## 0.1.20 +- Allow using `requests.auth.AuthBase` as authenticators instead of custom CDK authenticators. +- Implement Oauth2Authenticator, MultipleTokenAuthenticator and TokenAuthenticator authenticators. +- Add support for both legacy and requests native authenticator to HttpStream class. + ## 0.1.19 No longer prints full config files on validation error to prevent exposing secrets to log file: https://github.com/airbytehq/airbyte/pull/5879 diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py index 583e0cc66c92..044328d4008b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py @@ -1,5 +1,5 @@ # Initialize Streams Package from .exceptions import UserDefinedBackoffException -from .http import HttpStream +from .http import HttpStream, HttpSubStream -__all__ = ["HttpStream", "UserDefinedBackoffException"] +__all__ = ["HttpStream", "HttpSubStream", "UserDefinedBackoffException"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/core.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/core.py index 5db5cfea1f75..97bcc3b58645 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/core.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/core.py @@ -26,7 +26,10 @@ from abc import ABC, abstractmethod from typing import Any, Mapping +from deprecated import deprecated + +@deprecated(version="0.1.20", reason="Use requests.auth.AuthBase instead") class HttpAuthenticator(ABC): """ Base abstract class for various HTTP Authentication strategies. Authentication strategies are generally @@ -40,6 +43,7 @@ def get_auth_header(self) -> Mapping[str, Any]: """ +@deprecated(version="0.1.20", reason="Set `authenticator=None` instead") class NoAuth(HttpAuthenticator): def get_auth_header(self) -> Mapping[str, Any]: return {} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/oauth.py index d7799e25ab73..8e541c259dfd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/oauth.py @@ -27,22 +27,33 @@ import pendulum import requests +from deprecated import deprecated from .core import HttpAuthenticator +@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.Oauth2Authenticator instead") class Oauth2Authenticator(HttpAuthenticator): """ Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials. The generated access token is attached to each request via the Authorization header. """ - def __init__(self, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str, scopes: List[str] = None): + def __init__( + self, + token_refresh_endpoint: str, + client_id: str, + client_secret: str, + refresh_token: str, + scopes: List[str] = None, + refresh_access_token_headers: Mapping[str, Any] = None, + ): self.token_refresh_endpoint = token_refresh_endpoint self.client_secret = client_secret self.client_id = client_id self.refresh_token = refresh_token self.scopes = scopes + self.refresh_access_token_headers = refresh_access_token_headers self._token_expiry_date = pendulum.now().subtract(days=1) self._access_token = None @@ -81,7 +92,12 @@ def refresh_access_token(self) -> Tuple[str, int]: returns a tuple of (access_token, token_lifespan_in_seconds) """ try: - response = requests.request(method="POST", url=self.token_refresh_endpoint, data=self.get_refresh_request_body()) + response = requests.request( + method="POST", + url=self.token_refresh_endpoint, + data=self.get_refresh_request_body(), + headers=self.refresh_access_token_headers, + ) response.raise_for_status() response_json = response.json() return response_json["access_token"], response_json["expires_in"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/token.py index 294e19175d3e..64da6c61f8e1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/token.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/auth/token.py @@ -26,9 +26,12 @@ from itertools import cycle from typing import Any, List, Mapping +from deprecated import deprecated + from .core import HttpAuthenticator +@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.TokenAuthenticator instead") class TokenAuthenticator(HttpAuthenticator): def __init__(self, token: str, auth_method: str = "Bearer", auth_header: str = "Authorization"): self.auth_method = auth_method @@ -39,6 +42,7 @@ def get_auth_header(self) -> Mapping[str, Any]: return {self.auth_header: f"{self.auth_method} {self._token}"} +@deprecated(version="0.1.20", reason="Use airbyte_cdk.sources.streams.http.requests_native_auth.MultipleTokenAuthenticator instead") class MultipleTokenAuthenticator(HttpAuthenticator): def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"): self.auth_method = auth_method diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 83f1e00a0643..f480b7aab1b3 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -23,12 +23,16 @@ # +import os from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import requests +import vcr +import vcr.cassette as Cassette from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.core import Stream +from requests.auth import AuthBase from .auth.core import HttpAuthenticator, NoAuth from .exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException @@ -46,10 +50,49 @@ class HttpStream(Stream, ABC): source_defined_cursor = True # Most HTTP streams use a source defined cursor (i.e: the user can't configure it like on a SQL table) page_size = None # Use this variable to define page size for API http requests with pagination support - def __init__(self, authenticator: HttpAuthenticator = NoAuth()): - self._authenticator = authenticator + # TODO: remove legacy HttpAuthenticator authenticator references + def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None): self._session = requests.Session() + self._authenticator = NoAuth() + if isinstance(authenticator, AuthBase): + self._session.auth = authenticator + elif authenticator: + self._authenticator = authenticator + + if self.use_cache: + self.cache_file = self.request_cache() + # we need this attr to get metadata about cassettes, such as record play count, all records played, etc. + self.cassete = None + + @property + def cache_filename(self): + """ + Override if needed. Return the name of cache file + """ + return f"{self.name}.yml" + + @property + def use_cache(self): + """ + Override if needed. If True, all records will be cached. + """ + return False + + def request_cache(self) -> Cassette: + """ + Builds VCR instance. + It deletes file everytime we create it, normally should be called only once. + We can't use NamedTemporaryFile here because yaml serializer doesn't work well with empty files. + """ + + try: + os.remove(self.cache_filename) + except FileNotFoundError: + pass + + return vcr.use_cassette(self.cache_filename, record_mode="new_episodes", serializer="yaml") + @property @abstractmethod def url_base(self) -> str: @@ -315,7 +358,18 @@ def read_records( data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), ) request_kwargs = self.request_kwargs(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - response = self._send_request(request, request_kwargs) + + if self.use_cache: + # use context manager to handle and store cassette metadata + with self.cache_file as cass: + self.cassete = cass + # vcr tries to find records based on the request, if such records exist, return from cache file + # else make a request and save record in cache file + response = self._send_request(request, request_kwargs) + + else: + response = self._send_request(request, request_kwargs) + yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice) next_page_token = self.next_page_token(response) @@ -324,3 +378,29 @@ def read_records( # Always return an empty generator just in case no records were ever yielded yield from [] + + +class HttpSubStream(HttpStream, ABC): + def __init__(self, parent: HttpStream, **kwargs): + """ + :param parent: should be the instance of HttpStream class + """ + super().__init__(**kwargs) + self.parent = parent + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + + # iterate over all parent stream_slices + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + + # iterate over all parent records with current stream_slice + for record in parent_records: + yield {"parent": record} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/unit_test.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py similarity index 84% rename from airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/unit_test.py rename to airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py index b8a8150b507f..8b62c71c24da 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/unit_test.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/__init__.py @@ -22,6 +22,11 @@ # SOFTWARE. # +from .oauth import Oauth2Authenticator +from .token import MultipleTokenAuthenticator, TokenAuthenticator -def test_example_method(): - assert True +__all__ = [ + "Oauth2Authenticator", + "TokenAuthenticator", + "MultipleTokenAuthenticator", +] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py new file mode 100644 index 000000000000..ee90164a70e9 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -0,0 +1,104 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from typing import Any, List, Mapping, MutableMapping, Tuple + +import pendulum +import requests +from requests.auth import AuthBase + + +class Oauth2Authenticator(AuthBase): + """ + Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials. + The generated access token is attached to each request via the Authorization header. + """ + + def __init__( + self, + token_refresh_endpoint: str, + client_id: str, + client_secret: str, + refresh_token: str, + scopes: List[str] = None, + token_expiry_date: pendulum.datetime = None, + access_token_name: str = "access_token", + expires_in_name: str = "expires_in", + ): + self.token_refresh_endpoint = token_refresh_endpoint + self.client_secret = client_secret + self.client_id = client_id + self.refresh_token = refresh_token + self.scopes = scopes + self.access_token_name = access_token_name + self.expires_in_name = expires_in_name + + self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) + self._access_token = None + + def __call__(self, request): + request.headers.update(self.get_auth_header()) + return request + + def get_auth_header(self) -> Mapping[str, Any]: + return {"Authorization": f"Bearer {self.get_access_token()}"} + + def get_access_token(self): + if self.token_has_expired(): + t0 = pendulum.now() + token, expires_in = self.refresh_access_token() + self._access_token = token + self._token_expiry_date = t0.add(seconds=expires_in) + + return self._access_token + + def token_has_expired(self) -> bool: + return pendulum.now() > self._token_expiry_date + + def get_refresh_request_body(self) -> Mapping[str, Any]: + """Override to define additional parameters""" + payload: MutableMapping[str, Any] = { + "grant_type": "refresh_token", + "client_id": self.client_id, + "client_secret": self.client_secret, + "refresh_token": self.refresh_token, + } + + if self.scopes: + payload["scopes"] = self.scopes + + return payload + + def refresh_access_token(self) -> Tuple[str, int]: + """ + returns a tuple of (access_token, token_lifespan_in_seconds) + """ + try: + response = requests.request(method="POST", url=self.token_refresh_endpoint, data=self.get_refresh_request_body()) + response.raise_for_status() + response_json = response.json() + return response_json[self.access_token_name], response_json[self.expires_in_name] + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py new file mode 100644 index 000000000000..925962993fba --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/token.py @@ -0,0 +1,59 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from itertools import cycle +from typing import Any, List, Mapping + +from requests.auth import AuthBase + + +class MultipleTokenAuthenticator(AuthBase): + """ + Builds auth header, based on the list of tokens provided. + Auth header is changed per each `get_auth_header` call, using each token in cycle. + The token is attached to each request via the `auth_header` header. + """ + + def __init__(self, tokens: List[str], auth_method: str = "Bearer", auth_header: str = "Authorization"): + self.auth_method = auth_method + self.auth_header = auth_header + self._tokens = tokens + self._tokens_iter = cycle(self._tokens) + + def __call__(self, request): + request.headers.update(self.get_auth_header()) + return request + + def get_auth_header(self) -> Mapping[str, Any]: + return {self.auth_header: f"{self.auth_method} {next(self._tokens_iter)}"} + + +class TokenAuthenticator(MultipleTokenAuthenticator): + """ + Builds auth header, based on the token provided. + The token is attached to each request via the `auth_header` header. + """ + + def __init__(self, token: str, auth_method: str = "Bearer", auth_header: str = "Authorization"): + super().__init__([token], auth_method, auth_header) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py index 2cc15730342a..496d416b5b52 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py @@ -23,77 +23,20 @@ # +import importlib import json import os import pkgutil from typing import Any, ClassVar, Dict, Mapping, Tuple -import pkg_resources +import jsonref from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import ConnectorSpecification -from jsonschema import RefResolver, validate +from jsonschema import validate from jsonschema.exceptions import ValidationError from pydantic import BaseModel, Field -class JsonSchemaResolver: - """Helper class to expand $ref items in json schema""" - - def __init__(self, shared_schemas_path: str): - self._shared_refs = self._load_shared_schema_refs(shared_schemas_path) - - @staticmethod - def _load_shared_schema_refs(shared_schemas_path: str): - shared_file_names = [f.name for f in os.scandir(shared_schemas_path) if f.is_file()] - shared_schema_refs = {} - for shared_file in shared_file_names: - with open(os.path.join(shared_schemas_path, shared_file)) as data_file: - shared_schema_refs[shared_file] = json.load(data_file) - - return shared_schema_refs - - def _resolve_schema_references(self, schema: dict, resolver: RefResolver) -> dict: - if "$ref" in schema: - reference_path = schema.pop("$ref", None) - resolved = resolver.resolve(reference_path)[1] - schema.update(resolved) - return self._resolve_schema_references(schema, resolver) - - if "properties" in schema: - for k, val in schema["properties"].items(): - schema["properties"][k] = self._resolve_schema_references(val, resolver) - - if "patternProperties" in schema: - for k, val in schema["patternProperties"].items(): - schema["patternProperties"][k] = self._resolve_schema_references(val, resolver) - - if "items" in schema: - schema["items"] = self._resolve_schema_references(schema["items"], resolver) - - if "anyOf" in schema: - for i, element in enumerate(schema["anyOf"]): - schema["anyOf"][i] = self._resolve_schema_references(element, resolver) - - return schema - - def resolve(self, schema: dict, refs: Dict[str, dict] = None) -> dict: - """Resolves and replaces json-schema $refs with the appropriate dict. - Recursively walks the given schema dict, converting every instance - of $ref in a 'properties' structure with a resolved dict. - This modifies the input schema and also returns it. - Arguments: - schema: - the schema dict - refs: - a dict of which forms a store of referenced schemata - Returns: - schema - """ - refs = refs or {} - refs = {**self._shared_refs, **refs} - return self._resolve_schema_references(schema, RefResolver("", schema, store=refs)) - - class ResourceSchemaLoader: """JSONSchema loader from package resources""" @@ -124,10 +67,63 @@ def get_schema(self, name: str) -> dict: print(f"Invalid JSON file format for file {schema_filename}") raise - shared_schemas_folder = pkg_resources.resource_filename(self.package_name, "schemas/shared/") - if os.path.exists(shared_schemas_folder): - return JsonSchemaResolver(shared_schemas_folder).resolve(raw_schema) - return raw_schema + return self.__resolve_schema_references(raw_schema) + + def __resolve_schema_references(self, raw_schema: dict) -> dict: + """ + Resolve links to external references and move it to local "definitions" map. + :param raw_schema jsonschema to lookup for external links. + :return JSON serializable object with references without external dependencies. + """ + + class JsonFileLoader: + """ + Custom json file loader to resolve references to resources located in "shared" directory. + We need this for compatability with existing schemas cause all of them have references + pointing to shared_schema.json file instead of shared/shared_schema.json + """ + + def __init__(self, uri_base: str, shared: str): + self.shared = shared + self.uri_base = uri_base + + def __call__(self, uri: str) -> Dict[str, Any]: + uri = uri.replace(self.uri_base, f"{self.uri_base}/{self.shared}/") + return json.load(open(uri)) + + package = importlib.import_module(self.package_name) + base = os.path.dirname(package.__file__) + "/" + + def create_definitions(obj: dict, definitions: dict) -> Dict[str, Any]: + """ + Scan resolved schema and compose definitions section, also convert + jsonref.JsonRef object to JSON serializable dict. + :param obj - jsonschema object with ref field resovled. + :definitions - object for storing generated definitions. + :return JSON serializable object with references without external dependencies. + """ + if isinstance(obj, jsonref.JsonRef): + def_key = obj.__reference__["$ref"] + def_key = def_key.replace("#/definitions/", "").replace(".json", "_") + definition = create_definitions(obj.__subject__, definitions) + # Omit existance definitions for extenal resource since + # we dont need it anymore. + definition.pop("definitions", None) + definitions[def_key] = definition + return {"$ref": "#/definitions/" + def_key} + elif isinstance(obj, dict): + return {k: create_definitions(v, definitions) for k, v in obj.items()} + elif isinstance(obj, list): + return [create_definitions(item, definitions) for item in obj] + else: + return obj + + resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/shared"), base_uri=base) + definitions = {} + resolved = create_definitions(resolved, definitions) + if definitions: + resolved["definitions"] = definitions + return resolved def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: ConnectorSpecification, logger: AirbyteLogger): diff --git a/airbyte-cdk/python/docs/concepts/http-streams.md b/airbyte-cdk/python/docs/concepts/http-streams.md index 12fda0eca2cb..5bedb787c747 100644 --- a/airbyte-cdk/python/docs/concepts/http-streams.md +++ b/airbyte-cdk/python/docs/concepts/http-streams.md @@ -72,7 +72,12 @@ errors. It is not currently possible to specify a rate limit Airbyte should adhe When implementing [stream slicing](incremental-stream.md#streamstream_slices) in an `HTTPStream` each Slice is equivalent to a HTTP request; the stream will make one request per element returned by the `stream_slices` function. The current slice being read is passed into every other method in `HttpStream` e.g: `request_params`, `request_headers`, `path`, etc.. to be injected into a request. This allows you to dynamically determine the output of the `request_params`, `path`, and other functions to read the input slice and return the appropriate value. +### Caching + +When we are dealing with streams that depend on the results of another stream, we can use caching to write the data of the parent stream to a file in order to use this data when the child stream synchronizes, rather than performing a full HTTP request again. We can turn on caching by overriding use_cache property, and use HttpSubStream class as base class of child stream. + ### Network Adapter Keyword arguments + If you need to set any network-adapter keyword args on the outgoing HTTP requests such as `allow_redirects`, `stream`, `verify`, `cert`, etc.. override the `request_kwargs` method. Any option listed in [BaseAdapter.send](https://docs.python-requests.org/en/latest/api/#requests.adapters.BaseAdapter.send) can be returned as a keyword argument. diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 7e5223f33a26..47b58281d845 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -35,7 +35,7 @@ setup( name="airbyte-cdk", - version="0.1.18", + version="0.1.23", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -67,10 +67,13 @@ install_requires=[ "backoff", "jsonschema~=3.2.0", + "jsonref~=0.2", "pendulum", "pydantic~=1.6", "PyYAML~=5.4", "requests", + "vcrpy", + "Deprecated~=1.2", ], python_requires=">=3.7.0", extras_require={"dev": ["MyPy~=0.812", "pytest", "pytest-cov", "pytest-mock", "requests-mock"]}, diff --git a/airbyte-cdk/python/unit_tests/destinations/__init__.py b/airbyte-cdk/python/unit_tests/destinations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/singer/__init__.py b/airbyte-cdk/python/unit_tests/singer/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/__init__.py b/airbyte-cdk/python/unit_tests/sources/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/__init__.py b/airbyte-cdk/python/unit_tests/sources/streams/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/__init__.py b/airbyte-cdk/python/unit_tests/sources/streams/http/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/auth/__init__.py b/airbyte-cdk/python/unit_tests/sources/streams/http/auth/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py index 3e561af92acb..01d496b4f64e 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py @@ -25,9 +25,7 @@ import logging -import requests from airbyte_cdk.sources.streams.http.auth import MultipleTokenAuthenticator, NoAuth, Oauth2Authenticator, TokenAuthenticator -from requests import Response LOGGER = logging.getLogger(__name__) @@ -68,10 +66,11 @@ class TestOauth2Authenticator: Test class for OAuth2Authenticator. """ - refresh_endpoint = "refresh_end" + refresh_endpoint = "https://some_url.com/v1" client_id = "client_id" client_secret = "client_secret" refresh_token = "refresh_token" + refresh_access_token_headers = {"Header_1": "value 1", "Header_2": "value 2"} def test_get_auth_header_fresh(self, mocker): """ @@ -130,18 +129,23 @@ def test_refresh_request_body(self): } assert body == expected - def test_refresh_access_token(self, mocker): + def test_refresh_access_token(self, requests_mock): + mock_refresh_token_call = requests_mock.post( + TestOauth2Authenticator.refresh_endpoint, json={"access_token": "token", "expires_in": 10} + ) + oauth = Oauth2Authenticator( TestOauth2Authenticator.refresh_endpoint, TestOauth2Authenticator.client_id, TestOauth2Authenticator.client_secret, TestOauth2Authenticator.refresh_token, + refresh_access_token_headers=TestOauth2Authenticator.refresh_access_token_headers, ) - resp = Response() - resp.status_code = 200 - mocker.patch.object(requests, "request", return_value=resp) - mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": 1000}) token = oauth.refresh_access_token() - assert ("access_token", 1000) == token + assert ("token", 10) == token + for header in self.refresh_access_token_headers: + assert header in mock_refresh_token_call.last_request.headers + assert self.refresh_access_token_headers[header] == mock_refresh_token_call.last_request.headers[header] + assert mock_refresh_token_call.called diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/__init__.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py new file mode 100644 index 000000000000..f1a88dadc585 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -0,0 +1,164 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import logging + +import requests +from airbyte_cdk.sources.streams.http.requests_native_auth import MultipleTokenAuthenticator, Oauth2Authenticator, TokenAuthenticator +from requests import Response + +LOGGER = logging.getLogger(__name__) + + +def test_token_authenticator(): + """ + Should match passed in token, no matter how many times token is retrieved. + """ + token_auth = TokenAuthenticator(token="test-token") + header1 = token_auth.get_auth_header() + header2 = token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + token_auth(prepared_request) + + assert {"Authorization": "Bearer test-token"} == prepared_request.headers + assert {"Authorization": "Bearer test-token"} == header1 + assert {"Authorization": "Bearer test-token"} == header2 + + +def test_multiple_token_authenticator(): + multiple_token_auth = MultipleTokenAuthenticator(tokens=["token1", "token2"]) + header1 = multiple_token_auth.get_auth_header() + header2 = multiple_token_auth.get_auth_header() + header3 = multiple_token_auth.get_auth_header() + + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + multiple_token_auth(prepared_request) + + assert {"Authorization": "Bearer token2"} == prepared_request.headers + assert {"Authorization": "Bearer token1"} == header1 + assert {"Authorization": "Bearer token2"} == header2 + assert {"Authorization": "Bearer token1"} == header3 + + +class TestOauth2Authenticator: + """ + Test class for OAuth2Authenticator. + """ + + refresh_endpoint = "refresh_end" + client_id = "client_id" + client_secret = "client_secret" + refresh_token = "refresh_token" + + def test_get_auth_header_fresh(self, mocker): + """ + Should not retrieve new token if current token is valid. + """ + oauth = Oauth2Authenticator( + token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint, + client_id=TestOauth2Authenticator.client_id, + client_secret=TestOauth2Authenticator.client_secret, + refresh_token=TestOauth2Authenticator.refresh_token, + ) + + mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token", 1000)) + header = oauth.get_auth_header() + assert {"Authorization": "Bearer access_token"} == header + + def test_get_auth_header_expired(self, mocker): + """ + Should retrieve new token if current token is expired. + """ + oauth = Oauth2Authenticator( + token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint, + client_id=TestOauth2Authenticator.client_id, + client_secret=TestOauth2Authenticator.client_secret, + refresh_token=TestOauth2Authenticator.refresh_token, + ) + + expire_immediately = 0 + mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token_1", expire_immediately)) + oauth.get_auth_header() # Set the first expired token. + + valid_100_secs = 100 + mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token_2", valid_100_secs)) + header = oauth.get_auth_header() + assert {"Authorization": "Bearer access_token_2"} == header + + def test_refresh_request_body(self): + """ + Request body should match given configuration. + """ + scopes = ["scope1", "scope2"] + oauth = Oauth2Authenticator( + token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint, + client_id=TestOauth2Authenticator.client_id, + client_secret=TestOauth2Authenticator.client_secret, + refresh_token=TestOauth2Authenticator.refresh_token, + scopes=scopes, + ) + body = oauth.get_refresh_request_body() + expected = { + "grant_type": "refresh_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + "scopes": scopes, + } + assert body == expected + + def test_refresh_access_token(self, mocker): + oauth = Oauth2Authenticator( + token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint, + client_id=TestOauth2Authenticator.client_id, + client_secret=TestOauth2Authenticator.client_secret, + refresh_token=TestOauth2Authenticator.refresh_token, + ) + resp = Response() + resp.status_code = 200 + + mocker.patch.object(requests, "request", return_value=resp) + mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": 1000}) + token = oauth.refresh_access_token() + + assert ("access_token", 1000) == token + + def test_auth_call_method(self, mocker): + oauth = Oauth2Authenticator( + token_refresh_endpoint=TestOauth2Authenticator.refresh_endpoint, + client_id=TestOauth2Authenticator.client_id, + client_secret=TestOauth2Authenticator.client_secret, + refresh_token=TestOauth2Authenticator.refresh_token, + ) + + mocker.patch.object(Oauth2Authenticator, "refresh_access_token", return_value=("access_token", 1000)) + prepared_request = requests.PreparedRequest() + prepared_request.headers = {} + oauth(prepared_request) + + assert {"Authorization": "Bearer access_token"} == prepared_request.headers diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py index 84a53835243d..bb49a3c16a97 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py @@ -31,16 +31,19 @@ import pytest import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.auth import NoAuth +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator as HttpTokenAuthenticator from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator class StubBasicReadHttpStream(HttpStream): url_base = "https://test_base_url.com" primary_key = "" - def __init__(self): - super().__init__() + def __init__(self, **kwargs): + super().__init__(**kwargs) self.resp_counter = 1 def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -63,6 +66,24 @@ def parse_response( yield stubResp +def test_default_authenticator(): + stream = StubBasicReadHttpStream() + assert isinstance(stream.authenticator, NoAuth) + assert stream._session.auth is None + + +def test_requests_native_token_authenticator(): + stream = StubBasicReadHttpStream(authenticator=TokenAuthenticator("test-token")) + assert isinstance(stream.authenticator, NoAuth) + assert isinstance(stream._session.auth, TokenAuthenticator) + + +def test_http_token_authenticator(): + stream = StubBasicReadHttpStream(authenticator=HttpTokenAuthenticator("test-token")) + assert isinstance(stream.authenticator, HttpTokenAuthenticator) + assert stream._session.auth is None + + def test_request_kwargs_used(mocker, requests_mock): stream = StubBasicReadHttpStream() request_kwargs = {"cert": None, "proxies": "google.com"} @@ -350,3 +371,81 @@ def test_body_for_all_methods(self, mocker, requests_mock): assert response["body"] == self.data_body else: assert response["body"] is None + + +class CacheHttpStream(StubBasicReadHttpStream): + use_cache = True + + +class CacheHttpSubStream(HttpSubStream): + url_base = "https://example.com" + primary_key = "" + + def __init__(self, parent): + super().__init__(parent=parent) + + def parse_response(self, **kwargs) -> Iterable[Mapping]: + return [] + + def next_page_token(self, **kwargs) -> Optional[Mapping[str, Any]]: + return None + + def path(self, **kwargs) -> str: + return "" + + +def test_caching_filename(): + stream = CacheHttpStream() + assert stream.cache_filename == f"{stream.name}.yml" + + +def test_caching_cassettes_are_different(): + stream_1 = CacheHttpStream() + stream_2 = CacheHttpStream() + + assert stream_1.cache_file != stream_2.cache_file + + +def test_parent_attribute_exist(): + parent_stream = CacheHttpStream() + child_stream = CacheHttpSubStream(parent=parent_stream) + + assert child_stream.parent == parent_stream + + +def test_cache_response(mocker): + stream = CacheHttpStream() + mocker.patch.object(stream, "url_base", "https://google.com/") + list(stream.read_records(sync_mode=SyncMode.full_refresh)) + + with open(stream.cache_filename, "r") as f: + assert f.read() + + +class CacheHttpStreamWithSlices(CacheHttpStream): + paths = ["", "search"] + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f'{stream_slice.get("path")}' + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for path in self.paths: + yield {"path": path} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield response + + +def test_using_cache(mocker): + parent_stream = CacheHttpStreamWithSlices() + mocker.patch.object(parent_stream, "url_base", "https://google.com/") + + for _slice in parent_stream.stream_slices(): + list(parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice)) + + child_stream = CacheHttpSubStream(parent=parent_stream) + + for _slice in child_stream.stream_slices(sync_mode=SyncMode.full_refresh): + pass + + assert parent_stream.cassete.play_count != 0 diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py index de8d8b161308..b4713c200ed9 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py @@ -31,6 +31,7 @@ from collections.abc import Mapping from pathlib import Path +import jsonref from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader, check_config_against_spec_or_exit @@ -45,8 +46,6 @@ SCHEMAS_ROOT = "/".join(os.path.abspath(MODULE.__file__).split("/")[:-1]) / Path("schemas") -# TODO (sherif) refactor ResourceSchemaLoader to completely separate the functionality for reading data from the package. See https://github.com/airbytehq/airbyte/issues/3222 -# and the functionality for resolving schemas. See https://github.com/airbytehq/airbyte/issues/3222 @fixture(autouse=True, scope="session") def create_and_teardown_schemas_dir(): os.mkdir(SCHEMAS_ROOT) @@ -117,8 +116,9 @@ def test_shared_schemas_resolves(): "properties": { "str": {"type": "string"}, "int": {"type": "integer"}, - "obj": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}, + "obj": {"$ref": "#/definitions/shared_schema_"}, }, + "definitions": {"shared_schema_": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}}, } partial_schema = { @@ -135,3 +135,43 @@ def test_shared_schemas_resolves(): actual_schema = resolver.get_schema("complex_schema") assert actual_schema == expected_schema + + @staticmethod + def test_shared_schemas_resolves_nested(): + expected_schema = { + "type": ["null", "object"], + "properties": { + "str": {"type": "string"}, + "int": {"type": "integer"}, + "one_of": {"oneOf": [{"type": "string"}, {"$ref": "#/definitions/shared_schema_type_one"}]}, + "obj": {"$ref": "#/definitions/shared_schema_type_one"}, + }, + "definitions": {"shared_schema_type_one": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}}, + } + partial_schema = { + "type": ["null", "object"], + "properties": { + "str": {"type": "string"}, + "int": {"type": "integer"}, + "one_of": {"oneOf": [{"type": "string"}, {"$ref": "shared_schema.json#/definitions/type_one"}]}, + "obj": {"$ref": "shared_schema.json#/definitions/type_one"}, + }, + } + + referenced_schema = { + "definitions": { + "type_one": {"$ref": "shared_schema.json#/definitions/type_nested"}, + "type_nested": {"type": ["null", "object"], "properties": {"k1": {"type": "string"}}}, + } + } + + create_schema("complex_schema", partial_schema) + create_schema("shared/shared_schema", referenced_schema) + + resolver = ResourceSchemaLoader(MODULE_NAME) + + actual_schema = resolver.get_schema("complex_schema") + assert actual_schema == expected_schema + # Make sure generated schema is JSON serializable + assert json.dumps(actual_schema) + assert jsonref.JsonRef.replace_refs(actual_schema) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json index 9be2b8a0c312..42a5b5150bf8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "4816b78f-1489-44c1-9060-4b19d5fa9362", "name": "S3", "dockerRepository": "airbyte/destination-s3", - "dockerImageTag": "0.1.11", + "dockerImageTag": "0.1.12", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json index 1d14781e78f8..01515248aade 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/af7c921e-5892-4ff2-b6c1-4a5ab258fb7e.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", "name": "MeiliSearch", "dockerRepository": "airbyte/destination-meilisearch", - "dockerImageTag": "0.2.9", + "dockerImageTag": "0.2.10", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/meilisearch" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca8f6566-e555-4b40-943a-545bf123117a.json b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca8f6566-e555-4b40-943a-545bf123117a.json index 4d95d6eabbce..89078e4019d8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca8f6566-e555-4b40-943a-545bf123117a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/ca8f6566-e555-4b40-943a-545bf123117a.json @@ -2,6 +2,6 @@ "destinationDefinitionId": "ca8f6566-e555-4b40-943a-545bf123117a", "name": "Google Cloud Storage (GCS)", "dockerRepository": "airbyte/destination-gcs", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/destinations/gcs" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json index e3042f0e1be5..cd73c479261a 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/12928b32-bf0a-4f1e-964f-07e12e37153a.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "12928b32-bf0a-4f1e-964f-07e12e37153a", "name": "Mixpanel", "dockerRepository": "airbyte/source-mixpanel", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mixpanel", "icon": "mixpanel.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json index 35a8018b0655..1ce49774ec41 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "253487c0-2246-43ba-a21f-5116b20a2c50", "name": "Google Ads", "dockerRepository": "airbyte/source-google-ads", - "dockerImageTag": "0.1.10", + "dockerImageTag": "0.1.11", "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-ads" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json index 2069925a6d56..d4702301925c 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", "name": "Hubspot", "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.13", + "dockerImageTag": "0.1.15", "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", "icon": "hubspot.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/3981c999-bd7d-4afc-849b-e53dea90c948.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/3981c999-bd7d-4afc-849b-e53dea90c948.json new file mode 100644 index 000000000000..ec54561be514 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/3981c999-bd7d-4afc-849b-e53dea90c948.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "3981c999-bd7d-4afc-849b-e53dea90c948", + "name": "Lever Hiring", + "dockerRepository": "airbyte/source-lever-hiring", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/lever-hiring" +} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json index faa2d6e2626a..b70a95206760 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/435bb9a5-7887-4809-aa58-28c27df0d7ad.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "435bb9a5-7887-4809-aa58-28c27df0d7ad", "name": "MySQL", "dockerRepository": "airbyte/source-mysql", - "dockerImageTag": "0.4.3", + "dockerImageTag": "0.4.5", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mysql", "icon": "mysql.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json index fba118d7b66a..ba3528cd02e0 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/45d2e135-2ede-49e1-939f-3e3ec357a65e.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "45d2e135-2ede-49e1-939f-3e3ec357a65e", "name": "Recharge", "dockerRepository": "airbyte/source-recharge", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/sources/recharge" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json index d823494c608f..0c24266d3022 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/59f1e50a-331f-4f09-b3e8-2e8d4d355f44.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "59f1e50a-331f-4f09-b3e8-2e8d4d355f44", "name": "Greenhouse", "dockerRepository": "airbyte/source-greenhouse", - "dockerImageTag": "0.2.3", + "dockerImageTag": "0.2.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse", "icon": "greenhouse.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json index 24bf9de88414..3d84caee2caf 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", "name": "Shopify", "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.16", + "dockerImageTag": "0.1.18", "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b117307c-14b6-41aa-9422-947e34922962.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b117307c-14b6-41aa-9422-947e34922962.json index 41fe84ae36ed..bf4d411639e2 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b117307c-14b6-41aa-9422-947e34922962.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b117307c-14b6-41aa-9422-947e34922962.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "b117307c-14b6-41aa-9422-947e34922962", "name": "Salesforce", "dockerRepository": "airbyte/source-salesforce", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/salesforce", "icon": "salesforce.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json index e5a733278e00..e65a9e63751d 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/b5ea17b1-f170-46dc-bc31-cc744ca984c1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "b5ea17b1-f170-46dc-bc31-cc744ca984c1", "name": "Microsoft SQL Server (MSSQL)", "dockerRepository": "airbyte/source-mssql", - "dockerImageTag": "0.3.4", + "dockerImageTag": "0.3.6", "documentationUrl": "https://docs.airbyte.io/integrations/sources/mssql", "icon": "mssql.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c.json index 2a89ab26cc66..42b635e8e431 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c", "name": "BigQuery", "dockerRepository": "airbyte/source-bigquery", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/bigquery" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json index 551e47bc292d..2d1ebc5d195d 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c6b0a29e-1da9-4512-9002-7bfd0cba2246.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "c6b0a29e-1da9-4512-9002-7bfd0cba2246", "name": "Amazon Ads", "dockerRepository": "airbyte/source-amazon-ads", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/amazon-ads" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json index e99511fbad0a..a536e049b791 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", "name": "Intercom", "dockerRepository": "airbyte/source-intercom", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", "icon": "intercom.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json index efec2dea3ece..21942548be5b 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e094cb9a-26de-4645-8761-65c0c425d1de.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "e094cb9a-26de-4645-8761-65c0c425d1de", "name": "Stripe", "dockerRepository": "airbyte/source-stripe", - "dockerImageTag": "0.1.17", + "dockerImageTag": "0.1.18", "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", "icon": "stripe.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eb4c9e00-db83-4d63-a386-39cfa91012a8.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eb4c9e00-db83-4d63-a386-39cfa91012a8.json index 12f097057f2e..60282765f5b8 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eb4c9e00-db83-4d63-a386-39cfa91012a8.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eb4c9e00-db83-4d63-a386-39cfa91012a8.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "eb4c9e00-db83-4d63-a386-39cfa91012a8", "name": "Google Search Console (native)", "dockerRepository": "airbyte/source-google-search-console", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-search-console" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json index 473207582bdb..e9a59aedc93b 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/ef69ef6e-aa7f-4af1-a01d-ef775033524e.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "ef69ef6e-aa7f-4af1-a01d-ef775033524e", "name": "GitHub", "dockerRepository": "airbyte/source-github", - "dockerImageTag": "0.1.10", + "dockerImageTag": "0.2.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/github", "icon": "github.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json index 950d3300794e..aedaa4e4c700 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", "name": "Google Analytics v4", "dockerRepository": "airbyte/source-google-analytics-v4", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.3", "documentationUrl": "https://docs.airbyte.io/integrations/sources/source-google-analytics-v4", "icon": "google-analytics.svg" } diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json index 511f67d4ca82..a20b05d0de7f 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fa9f58c6-2d03-4237-aaa4-07d75e0c1396.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "fa9f58c6-2d03-4237-aaa4-07d75e0c1396", "name": "Amplitude", "dockerRepository": "airbyte/source-amplitude", - "dockerImageTag": "0.1.1", + "dockerImageTag": "0.1.2", "documentationUrl": "https://docs.airbyte.io/integrations/sources/amplitude" } diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index fb9ae85282a0..e8d413893af7 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -32,7 +32,7 @@ - destinationDefinitionId: ca8f6566-e555-4b40-943a-545bf123117a name: Google Cloud Storage (GCS) dockerRepository: airbyte/destination-gcs - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/destinations/gcs - destinationDefinitionId: 356668e2-7e34-47f3-a3b0-67a8a481b692 name: Google PubSub @@ -47,7 +47,7 @@ - destinationDefinitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 name: S3 dockerRepository: airbyte/destination-s3 - dockerImageTag: 0.1.11 + dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/destinations/s3 - destinationDefinitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc name: Redshift @@ -58,7 +58,7 @@ - destinationDefinitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e name: MeiliSearch dockerRepository: airbyte/destination-meilisearch - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 documentationUrl: https://docs.airbyte.io/integrations/destinations/meilisearch - destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 name: MySQL diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e5651ab25f5c..5f643ec2c2d4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -2,470 +2,563 @@ name: Amazon Seller Partner dockerRepository: airbyte/source-amazon-seller-partner dockerImageTag: 0.2.0 + sourceType: api documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-seller-partner - sourceDefinitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf name: Asana dockerRepository: airbyte/source-asana dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/asana + sourceType: api - sourceDefinitionId: 686473f1-76d9-4994-9cc7-9b13da46147c name: Chargebee dockerRepository: airbyte/source-chargebee dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/chargebee + sourceType: api - sourceDefinitionId: e2b40e36-aa0e-4bed-b41b-bcea6fa348b1 name: Exchange Rates Api dockerRepository: airbyte/source-exchange-rates dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/sources/exchangeratesapi icon: exchangeratesapi.svg + sourceType: api - sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 name: File dockerRepository: airbyte/source-file dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/file icon: file.svg + sourceType: file - sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 name: Google Ads dockerRepository: airbyte/source-google-ads - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads + sourceType: api - sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4 name: Google Adwords (Deprecated) dockerRepository: airbyte/source-google-adwords-singer dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/google-adwords icon: google-adwords.svg + sourceType: api - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e name: GitHub dockerRepository: airbyte/source-github - dockerImageTag: 0.1.10 + dockerImageTag: 0.2.1 documentationUrl: https://docs.airbyte.io/integrations/sources/github icon: github.svg + sourceType: api - sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 name: Microsoft SQL Server (MSSQL) dockerRepository: airbyte/source-mssql - dockerImageTag: 0.3.5 + dockerImageTag: 0.3.6 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg + sourceType: database - sourceDefinitionId: d8286229-c680-4063-8c59-23b9b391c700 name: Pipedrive dockerRepository: airbyte/source-pipedrive dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/pipedrive + sourceType: api - sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 name: Postgres dockerRepository: airbyte/source-postgres dockerImageTag: 0.3.11 documentationUrl: https://docs.airbyte.io/integrations/sources/postgres icon: postgresql.svg + sourceType: database - sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 name: Cockroachdb dockerRepository: airbyte/source-cockroachdb dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb + sourceType: database - sourceDefinitionId: af6d50ee-dddf-4126-a8ee-7faee990774f name: PostHog dockerRepository: airbyte/source-posthog dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/posthog + sourceType: api - sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01 name: Recurly dockerRepository: airbyte/source-recurly dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/sources/recurly icon: recurly.svg + sourceType: api - sourceDefinitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 name: S3 dockerRepository: airbyte/source-s3 dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/s3 + sourceType: file - sourceDefinitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 name: Sendgrid dockerRepository: airbyte/source-sendgrid dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/sendgrid icon: sendgrid.svg + sourceType: api - sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 name: Marketo dockerRepository: airbyte/source-marketo-singer dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/sources/marketo icon: marketo.svg + sourceType: api - sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 name: Google Sheets dockerRepository: airbyte/source-google-sheets dockerImageTag: 0.2.5 documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets icon: google-sheets.svg + sourceType: file - sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad name: MySQL dockerRepository: airbyte/source-mysql - dockerImageTag: 0.4.4 + dockerImageTag: 0.4.5 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg + sourceType: database - sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 name: Salesforce dockerRepository: airbyte/source-salesforce - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce icon: salesforce.svg + sourceType: api - sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de name: Stripe dockerRepository: airbyte/source-stripe - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg + sourceType: api - sourceDefinitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 name: Mailchimp dockerRepository: airbyte/source-mailchimp dockerImageTag: 0.2.8 documentationUrl: https://docs.airbyte.io/integrations/sources/mailchimp icon: mailchimp.svg + sourceType: api - sourceDefinitionId: 39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1 name: Google Analytics dockerRepository: airbyte/source-googleanalytics-singer dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/googleanalytics icon: google-analytics.svg + sourceType: api - sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 name: Google Analytics v4 dockerRepository: airbyte/source-google-analytics-v4 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/source-google-analytics-v4 icon: google-analytics.svg + sourceType: api - sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c name: Facebook Marketing dockerRepository: airbyte/source-facebook-marketing dockerImageTag: 0.2.17 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg + sourceType: api - sourceDefinitionId: 010eb12f-837b-4685-892d-0a39f76a98f5 name: Facebook Pages dockerRepository: airbyte/source-facebook-pages dockerImageTag: 0.1.0 documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-pages icon: facebook.svg + sourceType: api - sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c name: Hubspot dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.13 + dockerImageTag: 0.1.15 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg + sourceType: api - sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde name: Klaviyo dockerRepository: airbyte/source-klaviyo dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/klaviyo + sourceType: api - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 name: Shopify dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/sources/shopify + sourceType: api - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 name: Redshift dockerRepository: airbyte/source-redshift dockerImageTag: 0.3.2 documentationUrl: https://docs.airbyte.io/integrations/sources/redshift icon: redshift.svg + sourceType: database - sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 name: Twilio dockerRepository: airbyte/source-twilio dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/twilio + sourceType: api - sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 name: Freshdesk dockerRepository: airbyte/source-freshdesk dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk icon: freshdesk.svg + sourceType: api - sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 name: Greenhouse dockerRepository: airbyte/source-greenhouse - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse icon: greenhouse.svg + sourceType: api - sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 name: Zendesk Chat dockerRepository: airbyte/source-zendesk-chat dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg + sourceType: api - sourceDefinitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 name: Zendesk Support dockerRepository: airbyte/source-zendesk-support dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-support icon: zendesk.svg + sourceType: api - sourceDefinitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a name: Intercom dockerRepository: airbyte/source-intercom - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/intercom icon: intercom.svg + sourceType: api - sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 name: Jira dockerRepository: airbyte/source-jira dockerImageTag: 0.2.10 documentationUrl: https://docs.airbyte.io/integrations/sources/jira icon: jira.svg + sourceType: api - sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a name: Mixpanel dockerRepository: airbyte/source-mixpanel - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel icon: mixpanel.svg + sourceType: api - sourceDefinitionId: aea2fd0d-377d-465e-86c0-4fdc4f688e51 name: Zoom dockerRepository: airbyte/source-zoom-singer dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/sources/zoom icon: zoom.svg + sourceType: api - sourceDefinitionId: eaf50f04-21dd-4620-913b-2a83f5635227 name: Microsoft teams dockerRepository: airbyte/source-microsoft-teams dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.io/integrations/sources/microsoft-teams icon: microsoft-teams.svg + sourceType: api - sourceDefinitionId: 445831eb-78db-4b1f-8f1f-0d96ad8739e2 name: Drift dockerRepository: airbyte/source-drift dockerImageTag: 0.2.2 documentationUrl: https://docs.airbyte.io/integrations/sources/drift icon: drift.svg + sourceType: api - sourceDefinitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c name: Looker dockerRepository: airbyte/source-looker dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/sources/looker icon: looker.svg + sourceType: api - sourceDefinitionId: ed799e2b-2158-4c66-8da4-b40fe63bc72a name: Plaid dockerRepository: airbyte/source-plaid dockerImageTag: 0.2.1 documentationUrl: https://docs.airbyte.io/integrations/sources/plaid icon: plaid.svg + sourceType: api - sourceDefinitionId: 2af123bf-0aaf-4e0d-9784-cb497f23741a name: Appstore dockerRepository: airbyte/source-appstore-singer dockerImageTag: 0.2.4 documentationUrl: https://docs.airbyte.io/integrations/sources/appstore icon: appstore.svg + sourceType: api - sourceDefinitionId: d19ae824-e289-4b14-995a-0632eb46d246 name: Google Directory dockerRepository: airbyte/source-google-directory dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/google-directory + sourceType: api - sourceDefinitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 name: Instagram dockerRepository: airbyte/source-instagram dockerImageTag: 0.1.8 documentationUrl: https://hub.docker.com/r/airbyte/source-instagram + sourceType: api - sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 name: Gitlab dockerRepository: airbyte/source-gitlab dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/gitlab + sourceType: api - sourceDefinitionId: ed9dfefa-1bbc-419d-8c5e-4d78f0ef6734 name: Google Workspace Admin Reports dockerRepository: airbyte/source-google-workspace-admin-reports dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports + sourceType: api - sourceDefinitionId: d1aa448b-7c54-498e-ad95-263cbebcd2db name: Tempo dockerRepository: airbyte/source-tempo dockerImageTag: 0.2.3 documentationUrl: https://docs.airbyte.io/integrations/sources/tempo + sourceType: api - sourceDefinitionId: 8da67652-004c-11ec-9a03-0242ac130003 name: Trello dockerRepository: airbyte/source-trello dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/trello + sourceType: api - sourceDefinitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc name: Smartsheets dockerRepository: airbyte/source-smartsheets dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/smartsheets + sourceType: api - sourceDefinitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 name: Oracle DB dockerRepository: airbyte/source-oracle dockerImageTag: 0.3.4 documentationUrl: https://docs.airbyte.io/integrations/sources/oracle + sourceType: database - sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071 name: Zendesk Talk dockerRepository: airbyte/source-zendesk-talk dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-talk + sourceType: api - sourceDefinitionId: 29b409d9-30a5-4cc8-ad50-886eb846fea3 name: Quickbooks dockerRepository: airbyte/source-quickbooks-singer dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/quickbooks + sourceType: api - sourceDefinitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 name: Iterable dockerRepository: airbyte/source-iterable dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/iterable + sourceType: api - sourceDefinitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 name: PokeAPI dockerRepository: airbyte/source-pokeapi dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/pokeapi + sourceType: api - sourceDefinitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 name: Google Search Console dockerRepository: airbyte/source-google-search-console - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/google-search-console + sourceType: api - sourceDefinitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c name: ClickHouse dockerRepository: airbyte/source-clickhouse dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/clickhouse + sourceType: database - sourceDefinitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e name: Recharge dockerRepository: airbyte/source-recharge - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/recharge + sourceType: api - sourceDefinitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 name: Harvest dockerRepository: airbyte/source-harvest dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/harvest + sourceType: api - sourceDefinitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396 name: Amplitude dockerRepository: airbyte/source-amplitude - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/amplitude + sourceType: api - sourceDefinitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 name: Snowflake dockerRepository: airbyte/source-snowflake dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/snowflake + sourceType: database - sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 name: IBM Db2 dockerRepository: airbyte/source-db2 dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/db2 + sourceType: database - sourceDefinitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd name: Slack dockerRepository: airbyte/source-slack dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.io/integrations/sources/slack icon: slack.svg + sourceType: api - sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 name: AWS CloudTrail dockerRepository: airbyte/source-aws-cloudtrail dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/aws-cloudtrail + sourceType: api - sourceDefinitionId: c4cfaeda-c757-489a-8aba-859fb08b6970 name: US Census dockerRepository: airbyte/source-us-census dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/us-census + sourceType: api - sourceDefinitionId: 1d4fdb25-64fc-4569-92da-fcdca79a8372 name: Okta dockerRepository: airbyte/source-okta dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/okta + sourceType: api - sourceDefinitionId: badc5925-0485-42be-8caa-b34096cb71b5 name: Survey Monkey dockerRepository: airbyte/source-surveymonkey dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/surveymonkey + sourceType: api - sourceDefinitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 name: Square dockerRepository: airbyte/source-square dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/square + sourceType: api - sourceDefinitionId: 325e0640-e7b3-4e24-b823-3361008f603f name: Zendesk Sunshine dockerRepository: airbyte/source-zendesk-sunshine dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-sunshine + sourceType: api - sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 name: Paypal Transaction dockerRepository: airbyte/source-paypal-transaction dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/paypal-transaction + sourceType: api - sourceDefinitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7 name: Dixa dockerRepository: airbyte/source-dixa dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/dixa + sourceType: api - sourceDefinitionId: e7eff203-90bf-43e5-a240-19ea3056c474 name: Typeform dockerRepository: airbyte/source-typeform dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/typeform + sourceType: api - sourceDefinitionId: bb1a6d31-6879-4819-a2bd-3eed299ea8e2 name: Cart.com dockerRepository: airbyte/source-cart dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/cart + sourceType: api - sourceDefinitionId: d60a46d4-709f-4092-a6b7-2457f7d455f5 name: Prestashop dockerRepository: airbyte/source-prestashop dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/presta-shop + sourceType: api - sourceDefinitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c name: BigQuery dockerRepository: airbyte/source-bigquery - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/bigquery + sourceType: database - sourceDefinitionId: 90916976-a132-4ce9-8bce-82a03dd58788 name: BambooHR dockerRepository: airbyte/source-bamboo-hr dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/bamboo-hr + sourceType: api - sourceDefinitionId: 200330b2-ea62-4d11-ac6d-cfe3e3f8ab2b name: Snapchat Marketing dockerRepository: airbyte/source-snapchat-marketing dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/snapchat-marketing + sourceType: api - sourceDefinitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 name: Apify Dataset dockerRepository: airbyte/source-apify-dataset dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/apify-dataset + sourceType: api - sourceDefinitionId: 3dc3037c-5ce8-4661-adc2-f7a9e3c5ece5 name: Zuora dockerRepository: airbyte/source-zuora dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/zuora + sourceType: api - sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 name: Bing Ads dockerRepository: airbyte/source-bing-ads dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads + sourceType: api - sourceDefinitionId: dfffecb7-9a13-43e9-acdc-b92af7997ca9 name: Close.com dockerRepository: airbyte/source-close-com dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/close-com + sourceType: api - sourceDefinitionId: 59c5501b-9f95-411e-9269-7143c939adbd name: BigCommerce dockerRepository: airbyte/source-bigcommerce dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/bigcommerce + sourceType: api - sourceDefinitionId: 2fed2292-5586-480c-af92-9944e39fe12d name: Short.io dockerRepository: airbyte/source-shortio dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/shortio + sourceType: api - sourceDefinitionId: 63cea06f-1c75-458d-88fe-ad48c7cb27fd name: Braintree dockerRepository: airbyte/source-braintree dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/braintree icon: braintree.svg + sourceType: api - sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 name: Amazon Ads dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads + sourceType: api - sourceDefinitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e name: MongoDb dockerRepository: airbyte/source-mongodb-v2 dockerImageTag: 0.1.0 documentationUrl: https://docs.airbyte.io/integrations/sources/mongodb-v2 icon: mongodb.svg + sourceType: database +- sourceDefinitionId: 3981c999-bd7d-4afc-849b-e53dea90c948 + name: Lever Hiring + dockerRepository: airbyte/source-lever-hiring + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/lever-hiring + sourceType: api diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java index 37ceadfba918..40e2a29c5cf0 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java @@ -26,6 +26,7 @@ import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.Set; public interface Configs { @@ -58,6 +59,12 @@ public interface Configs { String getConfigDatabaseUrl(); + String getSecretStoreGcpProjectId(); + + String getSecretStoreGcpCredentials(); + + String getSecretStoreForConfigs(); + boolean runDatabaseMigrationOnStartup(); int getMaxSyncJobAttempts(); @@ -84,6 +91,8 @@ public interface Configs { List getWorkerPodTolerations(); + Map getWorkerNodeSelectors(); + MaxWorkersConfig getMaxWorkers(); String getTemporalHost(); diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java index fb121f05eab2..09455b956811 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -69,6 +69,7 @@ public class EnvConfigs implements Configs { public static final String RUN_DATABASE_MIGRATION_ON_STARTUP = "RUN_DATABASE_MIGRATION_ON_STARTUP"; public static final String WEBAPP_URL = "WEBAPP_URL"; public static final String WORKER_POD_TOLERATIONS = "WORKER_POD_TOLERATIONS"; + public static final String WORKER_POD_NODE_SELECTORS = "WORKER_POD_NODE_SELECTORS"; public static final String MAX_SYNC_JOB_ATTEMPTS = "MAX_SYNC_JOB_ATTEMPTS"; public static final String MAX_SYNC_TIMEOUT_DAYS = "MAX_SYNC_TIMEOUT_DAYS"; private static final String MINIMUM_WORKSPACE_RETENTION_DAYS = "MINIMUM_WORKSPACE_RETENTION_DAYS"; @@ -92,6 +93,9 @@ public class EnvConfigs implements Configs { private static final String DEFAULT_KUBE_NAMESPACE = "default"; private static final String DEFAULT_RESOURCE_REQUIREMENT_CPU = null; private static final String DEFAULT_RESOURCE_REQUIREMENT_MEMORY = null; + private static final String SECRET_STORE_GCP_PROJECT_ID = "SECRET_STORE_GCP_PROJECT_ID"; + private static final String SECRET_STORE_GCP_CREDENTIALS = "SECRET_STORE_GCP_CREDENTIALS"; + private static final String SECRET_STORE_FOR_CONFIGS = "SECRET_STORE_CONFIGS_ENABLE"; private static final long DEFAULT_MINIMUM_WORKSPACE_RETENTION_DAYS = 1; private static final long DEFAULT_MAXIMUM_WORKSPACE_RETENTION_DAYS = 60; private static final long DEFAULT_MAXIMUM_WORKSPACE_SIZE_MB = 5000; @@ -196,6 +200,21 @@ public String getConfigDatabaseUrl() { return getEnvOrDefault(CONFIG_DATABASE_URL, getDatabaseUrl()); } + @Override + public String getSecretStoreGcpCredentials() { + return getEnv(SECRET_STORE_GCP_CREDENTIALS); + } + + @Override + public String getSecretStoreGcpProjectId() { + return getEnv(SECRET_STORE_GCP_PROJECT_ID); + } + + @Override + public String getSecretStoreForConfigs() { + return getEnv(SECRET_STORE_FOR_CONFIGS); + } + @Override public boolean runDatabaseMigrationOnStartup() { return getEnvOrDefault(RUN_DATABASE_MIGRATION_ON_STARTUP, true); @@ -311,6 +330,25 @@ public List getWorkerPodTolerations() { .collect(Collectors.toList()); } + /** + * Returns a map of node selectors from its own environment variable. The value of the env is a + * string that represents one or more node selector labels. Each kv-pair is separated by a `,` + *

+ * For example:- The following represents two node selectors + *

+ * airbyte=server,type=preemptive + * + * @return map containing kv pairs of node selectors + */ + @Override + public Map getWorkerNodeSelectors() { + return Splitter.on(",") + .splitToStream(getEnvOrDefault(WORKER_POD_NODE_SELECTORS, "")) + .filter(s -> !Strings.isNullOrEmpty(s) && s.contains("=")) + .map(s -> s.split("=")) + .collect(Collectors.toMap(s -> s[0], s -> s[1])); + } + @Override public MaxWorkersConfig getMaxWorkers() { return new MaxWorkersConfig( @@ -424,7 +462,7 @@ private T getEnvOrDefault(final String key, final T defaultValue, final Func if (value != null && !value.isEmpty()) { return parser.apply(value); } else { - LOGGER.info("{} not found or empty, defaulting to {}", key, isSecret ? "*****" : defaultValue); + LOGGER.info("Using default value for environment variable {}: '{}'", key, isSecret ? "*****" : defaultValue); return defaultValue; } } diff --git a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java index c69079775e60..2a4def16ea3b 100644 --- a/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java +++ b/airbyte-config/models/src/main/java/io/airbyte/config/helpers/LogClientSingleton.java @@ -91,7 +91,6 @@ public static File getServerLogFile(Configs configs) { } var logConfigs = new LogConfigDelegator(configs); - createCloudClientIfNull(logConfigs); var cloudLogPath = APP_LOGGING_CLOUD_PREFIX + logPathBase; try { return logClient.downloadCloudLog(logConfigs, cloudLogPath); @@ -107,7 +106,6 @@ public static File getSchedulerLogFile(Configs configs) { } var logConfigs = new LogConfigDelegator(configs); - createCloudClientIfNull(logConfigs); var cloudLogPath = APP_LOGGING_CLOUD_PREFIX + logPathBase; try { return logClient.downloadCloudLog(logConfigs, cloudLogPath); @@ -122,7 +120,6 @@ public static List getJobLogFile(Configs configs, Path logPath) throws I } var logConfigs = new LogConfigDelegator(configs); - createCloudClientIfNull(logConfigs); var cloudLogPath = JOB_LOGGING_CLOUD_PREFIX + logPath; return logClient.tailCloudLog(logConfigs, cloudLogPath, LOG_TAIL_SIZE); } @@ -136,33 +133,38 @@ public static void deleteLogs(Configs configs, String logPath) { throw new NotImplementedException("Local log deletes not supported."); } var logConfigs = new LogConfigDelegator(configs); - createCloudClientIfNull(logConfigs); var cloudLogPath = JOB_LOGGING_CLOUD_PREFIX + logPath; logClient.deleteLogs(logConfigs, cloudLogPath); } public static void setJobMdc(Path path) { - if (shouldUseLocalLogs(new EnvConfigs())) { + var configs = new EnvConfigs(); + if (shouldUseLocalLogs(configs)) { LOGGER.debug("Setting docker job mdc"); MDC.put(LogClientSingleton.JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); } else { LOGGER.debug("Setting kube job mdc"); + var logConfigs = new LogConfigDelegator(configs); + createCloudClientIfNull(logConfigs); MDC.put(LogClientSingleton.CLOUD_JOB_LOG_PATH_MDC_KEY, path.resolve(LogClientSingleton.LOG_FILENAME).toString()); } } public static void setWorkspaceMdc(Path path) { - if (shouldUseLocalLogs(new EnvConfigs())) { + var configs = new EnvConfigs(); + if (shouldUseLocalLogs(configs)) { LOGGER.debug("Setting docker workspace mdc"); MDC.put(LogClientSingleton.WORKSPACE_MDC_KEY, path.toString()); } else { LOGGER.debug("Setting kube workspace mdc"); + var logConfigs = new LogConfigDelegator(configs); + createCloudClientIfNull(logConfigs); MDC.put(LogClientSingleton.CLOUD_WORKSPACE_MDC_KEY, path.toString()); } } private static boolean shouldUseLocalLogs(Configs configs) { - return configs.getWorkerEnvironment().equals(WorkerEnvironment.DOCKER) || CloudLogs.hasEmptyConfigs(new LogConfigDelegator(configs)); + return configs.getWorkerEnvironment().equals(WorkerEnvironment.DOCKER); } private static void createCloudClientIfNull(LogConfigs configs) { diff --git a/airbyte-config/models/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-config/models/src/main/resources/types/StandardSourceDefinition.yaml index 9afb01c5d426..f48f73a6fa1f 100644 --- a/airbyte-config/models/src/main/resources/types/StandardSourceDefinition.yaml +++ b/airbyte-config/models/src/main/resources/types/StandardSourceDefinition.yaml @@ -10,7 +10,7 @@ required: - dockerRepository - dockerImageTag - documentationUrl -additionalProperties: false +additionalProperties: true properties: sourceDefinitionId: type: string @@ -25,3 +25,10 @@ properties: type: string icon: type: string + sourceType: + type: string + enum: + - api + - file + - database + - custom diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java index 81db2ff7fb05..5deed975c1d2 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/EnvConfigsTest.java @@ -28,6 +28,7 @@ import java.nio.file.Paths; import java.util.List; +import java.util.Map; import java.util.function.Function; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -204,4 +205,22 @@ void testWorkerPodTolerations() { new WorkerPodToleration("airbyte-server", "NoSchedule", "true", "Equals"))); } + @Test + void testWorkerPodNodeSelectors() { + when(function.apply(EnvConfigs.WORKER_POD_NODE_SELECTORS)).thenReturn(null); + Assertions.assertEquals(config.getWorkerNodeSelectors(), Map.of()); + + when(function.apply(EnvConfigs.WORKER_POD_NODE_SELECTORS)).thenReturn(",,,"); + Assertions.assertEquals(config.getWorkerNodeSelectors(), Map.of()); + + when(function.apply(EnvConfigs.WORKER_POD_NODE_SELECTORS)).thenReturn("key=k,,;$%&^#"); + Assertions.assertEquals(config.getWorkerNodeSelectors(), Map.of("key", "k")); + + when(function.apply(EnvConfigs.WORKER_POD_NODE_SELECTORS)).thenReturn("one=two"); + Assertions.assertEquals(config.getWorkerNodeSelectors(), Map.of("one", "two")); + + when(function.apply(EnvConfigs.WORKER_POD_NODE_SELECTORS)).thenReturn("airbyte=server,something=nothing"); + Assertions.assertEquals(config.getWorkerNodeSelectors(), Map.of("airbyte", "server", "something", "nothing")); + } + } diff --git a/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java b/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsClientTest.java similarity index 60% rename from airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java rename to airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsClientTest.java index 2e9a077ebb8f..18e55209b07b 100644 --- a/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsTest.java +++ b/airbyte-config/models/src/test/java/io/airbyte/config/helpers/CloudLogsClientTest.java @@ -25,11 +25,59 @@ package io.airbyte.config.helpers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -public class CloudLogsTest { +public class CloudLogsClientTest { + + @Nested + class CloudLogClientMissingConfiguration { + + @Test + public void testMinio() { + var configs = Mockito.mock(LogConfigs.class); + // Mising bucket. + Mockito.when(configs.getS3MinioEndpoint()).thenReturn("minio-endpoint"); + Mockito.when(configs.getAwsAccessKey()).thenReturn("access-key"); + Mockito.when(configs.getAwsSecretAccessKey()).thenReturn("access-key-secret"); + Mockito.when(configs.getS3LogBucket()).thenReturn(""); + Mockito.when(configs.getS3LogBucketRegion()).thenReturn(""); + + assertThrows(RuntimeException.class, () -> CloudLogs.createCloudLogClient(configs)); + } + + @Test + public void testAws() { + var configs = Mockito.mock(LogConfigs.class); + // Missing bucket and access key. + Mockito.when(configs.getS3MinioEndpoint()).thenReturn(""); + Mockito.when(configs.getAwsAccessKey()).thenReturn(""); + Mockito.when(configs.getAwsSecretAccessKey()).thenReturn("access-key-secret"); + Mockito.when(configs.getS3LogBucket()).thenReturn(""); + Mockito.when(configs.getS3LogBucketRegion()).thenReturn(""); + + assertThrows(RuntimeException.class, () -> CloudLogs.createCloudLogClient(configs)); + } + + @Test + public void testGcs() { + var configs = Mockito.mock(LogConfigs.class); + Mockito.when(configs.getAwsAccessKey()).thenReturn(""); + Mockito.when(configs.getAwsSecretAccessKey()).thenReturn(""); + Mockito.when(configs.getS3LogBucket()).thenReturn(""); + Mockito.when(configs.getS3LogBucketRegion()).thenReturn(""); + + // Missing bucket. + Mockito.when(configs.getGcpStorageBucket()).thenReturn(""); + Mockito.when(configs.getGoogleApplicationCredentials()).thenReturn("path/to/google/secret"); + + assertThrows(RuntimeException.class, () -> CloudLogs.createCloudLogClient(configs)); + } + + } @Test public void createCloudLogClientTestMinio() { diff --git a/airbyte-config/persistence/build.gradle b/airbyte-config/persistence/build.gradle index 40bb4cce193d..f10ad625baef 100644 --- a/airbyte-config/persistence/build.gradle +++ b/airbyte-config/persistence/build.gradle @@ -1,12 +1,18 @@ +plugins { + id 'java-library' + id 'airbyte-integration-test-java' +} + dependencies { implementation group: 'commons-io', name: 'commons-io', version: '2.7' implementation project(':airbyte-db:lib') implementation project(':airbyte-db:jooq') - implementation project(':airbyte-config:models') implementation project(':airbyte-protocol:models') + implementation project(':airbyte-config:models') implementation project(':airbyte-config:init') implementation project(':airbyte-json-validation') - + implementation 'com.google.cloud:google-cloud-secretmanager:1.7.2' testImplementation "org.testcontainers:postgresql:1.15.1" + integrationTestJavaImplementation project(':airbyte-config:persistence') } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java index 9c9a2f862dc3..c29b84667b56 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigRepository.java @@ -44,6 +44,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.stream.Stream; @@ -188,7 +189,12 @@ public DestinationConnection getDestinationConnection(final UUID destinationId) return persistence.getConfig(ConfigSchema.DESTINATION_CONNECTION, destinationId.toString(), DestinationConnection.class); } - public void writeDestinationConnection(final DestinationConnection destinationConnection) throws JsonValidationException, IOException { + public void writeDestinationConnection(final DestinationConnection destinationConnection, final ConnectorSpecification connectorSpecification) + throws JsonValidationException, IOException { + // actual validation is only for sanity checking + final JsonSchemaValidator validator = new JsonSchemaValidator(); + validator.ensure(connectorSpecification.getConnectionSpecification(), destinationConnection.getConfiguration()); + persistence.writeConfig(ConfigSchema.DESTINATION_CONNECTION, destinationConnection.getDestinationId().toString(), destinationConnection); } @@ -225,6 +231,17 @@ public SourceOAuthParameter getSourceOAuthParams(final UUID SourceOAuthParameter return persistence.getConfig(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameterId.toString(), SourceOAuthParameter.class); } + public Optional getSourceOAuthParamByDefinitionIdOptional(final UUID workspaceId, final UUID sourceDefinitionId) + throws JsonValidationException, IOException { + for (final SourceOAuthParameter oAuthParameter : listSourceOAuthParam()) { + if (sourceDefinitionId.equals(oAuthParameter.getSourceDefinitionId()) && + Objects.equals(workspaceId, oAuthParameter.getWorkspaceId())) { + return Optional.of(oAuthParameter); + } + } + return Optional.empty(); + } + public void writeSourceOAuthParam(final SourceOAuthParameter SourceOAuthParameter) throws JsonValidationException, IOException { persistence.writeConfig(ConfigSchema.SOURCE_OAUTH_PARAM, SourceOAuthParameter.getOauthParameterId().toString(), SourceOAuthParameter); } @@ -238,6 +255,18 @@ public DestinationOAuthParameter getDestinationOAuthParams(final UUID destinatio return persistence.getConfig(ConfigSchema.DESTINATION_OAUTH_PARAM, destinationOAuthParameterId.toString(), DestinationOAuthParameter.class); } + public Optional getDestinationOAuthParamByDefinitionIdOptional(final UUID workspaceId, + final UUID destinationDefinitionId) + throws JsonValidationException, IOException { + for (final DestinationOAuthParameter oAuthParameter : listDestinationOAuthParam()) { + if (destinationDefinitionId.equals(oAuthParameter.getDestinationDefinitionId()) && + Objects.equals(workspaceId, oAuthParameter.getWorkspaceId())) { + return Optional.of(oAuthParameter); + } + } + return Optional.empty(); + } + public void writeDestinationOAuthParam(final DestinationOAuthParameter destinationOAuthParameter) throws JsonValidationException, IOException { persistence.writeConfig(ConfigSchema.DESTINATION_OAUTH_PARAM, destinationOAuthParameter.getOauthParameterId().toString(), destinationOAuthParameter); diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java index 137f3d3d0233..37e500eb797a 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/DatabaseConfigPersistence.java @@ -30,12 +30,16 @@ import static org.jooq.impl.DSL.select; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.config.AirbyteConfig; import io.airbyte.config.ConfigSchema; import io.airbyte.config.ConfigSchemaMigrationSupport; +import io.airbyte.config.Configs; import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.db.Database; @@ -74,17 +78,31 @@ public DatabaseConfigPersistence(Database database) { } /** - * Load or update the configs from the seed. + * If this is a migration deployment from an old version that relies on file system config + * persistence, copy the existing configs from local files. */ - @Override - public void loadData(ConfigPersistence seedConfigPersistence) throws IOException { + public void migrateFileConfigs(Configs serverConfigs) throws IOException { database.transaction(ctx -> { - boolean isInitialized = ctx.fetchExists(select().from(AIRBYTE_CONFIGS).where()); + final boolean isInitialized = ctx.fetchExists(AIRBYTE_CONFIGS); if (isInitialized) { - updateConfigsFromSeed(ctx, seedConfigPersistence); - } else { - copyConfigsFromSeed(ctx, seedConfigPersistence); + return null; + } + + final boolean hasExistingFileConfigs = FileSystemConfigPersistence.hasExistingConfigs(serverConfigs.getConfigRoot()); + if (hasExistingFileConfigs) { + LOGGER.info("Load existing local config directory into configs database"); + ConfigPersistence fileSystemPersistence = new FileSystemConfigPersistence(serverConfigs.getConfigRoot()); + copyConfigsFromSeed(ctx, fileSystemPersistence); } + + return null; + }); + } + + @Override + public void loadData(ConfigPersistence seedConfigPersistence) throws IOException { + database.transaction(ctx -> { + updateConfigsFromSeed(ctx, seedConfigPersistence); return null; }); } @@ -239,7 +257,7 @@ int updateConfigRecord(DSLContext ctx, OffsetDateTime timestamp, String configTy @VisibleForTesting void copyConfigsFromSeed(DSLContext ctx, ConfigPersistence seedConfigPersistence) throws SQLException { - LOGGER.info("Loading data to config database..."); + LOGGER.info("Loading seed data to config database..."); Map> seedConfigs; try { @@ -262,19 +280,21 @@ void copyConfigsFromSeed(DSLContext ctx, ConfigPersistence seedConfigPersistence static class ConnectorInfo { + final String definitionId; + final JsonNode definition; final String dockerRepository; - final String connectorDefinitionId; final String dockerImageTag; - private ConnectorInfo(String dockerRepository, String connectorDefinitionId, String dockerImageTag) { - this.dockerRepository = dockerRepository; - this.connectorDefinitionId = connectorDefinitionId; - this.dockerImageTag = dockerImageTag; + ConnectorInfo(String definitionId, JsonNode definition) { + this.definitionId = definitionId; + this.definition = definition; + this.dockerRepository = definition.get("dockerRepository").asText(); + this.dockerImageTag = definition.get("dockerImageTag").asText(); } @Override public String toString() { - return String.format("%s: %s (%s)", dockerRepository, dockerImageTag, connectorDefinitionId); + return String.format("%s: %s (%s)", dockerRepository, dockerImageTag, definitionId); } } @@ -293,7 +313,7 @@ private ConnectorCounter(int newCount, int updateCount) { @VisibleForTesting void updateConfigsFromSeed(DSLContext ctx, ConfigPersistence seedConfigPersistence) throws SQLException { - LOGGER.info("Config database has been initialized; updating connector definitions from the seed if necessary..."); + LOGGER.info("Updating connector definitions from the seed if necessary..."); try { Set connectorRepositoriesInUse = getConnectorRepositoriesInUse(ctx); @@ -331,41 +351,91 @@ void updateConfigsFromSeed(DSLContext ctx, ConfigPersistence seedConfigPersisten * will not be updated. This is necessary because the new connector version may not be * backward compatible. */ - private ConnectorCounter updateConnectorDefinitions(DSLContext ctx, - OffsetDateTime timestamp, - AirbyteConfig configType, - List latestDefinitions, - Set connectorRepositoriesInUse, - Map connectorRepositoryToIdVersionMap) + @VisibleForTesting + ConnectorCounter updateConnectorDefinitions(DSLContext ctx, + OffsetDateTime timestamp, + AirbyteConfig configType, + List latestDefinitions, + Set connectorRepositoriesInUse, + Map connectorRepositoryToIdVersionMap) throws IOException { int newCount = 0; int updatedCount = 0; - for (T latestDefinition : latestDefinitions) { - JsonNode configJson = Jsons.jsonNode(latestDefinition); - String repository = configJson.get("dockerRepository").asText(); - if (connectorRepositoriesInUse.contains(repository)) { - LOGGER.info("Connector {} is in use; skip updating", repository); - continue; - } + for (T definition : latestDefinitions) { + JsonNode latestDefinition = Jsons.jsonNode(definition); + String repository = latestDefinition.get("dockerRepository").asText(); + + // Add new connector if (!connectorRepositoryToIdVersionMap.containsKey(repository)) { - LOGGER.info("Adding new connector {}: {}", repository, configJson); - newCount += insertConfigRecord(ctx, timestamp, configType.name(), configJson, configType.getIdFieldName()); + LOGGER.info("Adding new connector {}: {}", repository, latestDefinition); + newCount += insertConfigRecord(ctx, timestamp, configType.name(), latestDefinition, configType.getIdFieldName()); continue; } ConnectorInfo connectorInfo = connectorRepositoryToIdVersionMap.get(repository); - String latestImageTag = configJson.get("dockerImageTag").asText(); - if (!latestImageTag.equals(connectorInfo.dockerImageTag)) { + JsonNode currentDefinition = connectorInfo.definition; + Set newFields = getNewFields(currentDefinition, latestDefinition); + + // Process connector in use + if (connectorRepositoriesInUse.contains(repository)) { + if (newFields.size() == 0) { + LOGGER.info("Connector {} is in use and has all fields; skip updating", repository); + } else { + // Add new fields to the connector definition + JsonNode definitionToUpdate = getDefinitionWithNewFields(currentDefinition, latestDefinition, newFields); + LOGGER.info("Connector {} has new fields: {}", repository, String.join(", ", newFields)); + updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), definitionToUpdate, connectorInfo.definitionId); + } + continue; + } + + // Process unused connector + String latestImageTag = latestDefinition.get("dockerImageTag").asText(); + if (hasNewVersion(connectorInfo.dockerImageTag, latestImageTag)) { + // Update connector to the latest version LOGGER.info("Connector {} needs update: {} vs {}", repository, connectorInfo.dockerImageTag, latestImageTag); - updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), configJson, connectorInfo.connectorDefinitionId); + updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), latestDefinition, connectorInfo.definitionId); + } else if (newFields.size() > 0) { + // Add new fields to the connector definition + JsonNode definitionToUpdate = getDefinitionWithNewFields(currentDefinition, latestDefinition, newFields); + LOGGER.info("Connector {} has new fields: {}", repository, String.join(", ", newFields)); + updatedCount += updateConfigRecord(ctx, timestamp, configType.name(), definitionToUpdate, connectorInfo.definitionId); } else { LOGGER.info("Connector {} does not need update: {}", repository, connectorInfo.dockerImageTag); } } + return new ConnectorCounter(newCount, updatedCount); } + static boolean hasNewVersion(String currentVersion, String latestVersion) { + try { + return new AirbyteVersion(latestVersion).patchVersionCompareTo(new AirbyteVersion(currentVersion)) > 0; + } catch (Exception e) { + LOGGER.error("Failed to check version: {} vs {}", currentVersion, latestVersion); + return false; + } + } + + /** + * @return new fields from the latest definition + */ + static Set getNewFields(JsonNode currentDefinition, JsonNode latestDefinition) { + Set currentFields = MoreIterators.toSet(currentDefinition.fieldNames()); + Set latestFields = MoreIterators.toSet(latestDefinition.fieldNames()); + return Sets.difference(latestFields, currentFields); + } + + /** + * @return a clone of the current definition with the new fields from the latest definition. + */ + static JsonNode getDefinitionWithNewFields(JsonNode currentDefinition, JsonNode latestDefinition, Set newFields) { + ObjectNode currentClone = (ObjectNode) Jsons.clone(currentDefinition); + newFields.forEach(field -> currentClone.set(field, latestDefinition.get(field))); + return currentClone; + } + /** * @return A map about current connectors (both source and destination). It maps from connector * repository to its definition id and docker image tag. We identify a connector by its @@ -374,21 +444,21 @@ private ConnectorCounter updateConnectorDefinitions(DSLContext ctx, */ @VisibleForTesting Map getConnectorRepositoryToInfoMap(DSLContext ctx) { + Field configField = field("config_blob", SQLDataType.JSONB).as("definition"); Field repoField = field("config_blob ->> 'dockerRepository'", SQLDataType.VARCHAR).as("repository"); - Field versionField = field("config_blob ->> 'dockerImageTag'", SQLDataType.VARCHAR).as("version"); - return ctx.select(AIRBYTE_CONFIGS.CONFIG_ID, repoField, versionField) + return ctx.select(AIRBYTE_CONFIGS.CONFIG_ID, repoField, configField) .from(AIRBYTE_CONFIGS) .where(AIRBYTE_CONFIGS.CONFIG_TYPE.in(ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), ConfigSchema.STANDARD_DESTINATION_DEFINITION.name())) .fetch().stream() .collect(Collectors.toMap( row -> row.getValue(repoField), - row -> new ConnectorInfo(row.getValue(repoField), row.getValue(AIRBYTE_CONFIGS.CONFIG_ID), row.getValue(versionField)), + row -> new ConnectorInfo(row.getValue(AIRBYTE_CONFIGS.CONFIG_ID), Jsons.deserialize(row.getValue(configField).data())), // when there are duplicated connector definitions, return the latest one (c1, c2) -> { AirbyteVersion v1 = new AirbyteVersion(c1.dockerImageTag); AirbyteVersion v2 = new AirbyteVersion(c2.dockerImageTag); LOGGER.warn("Duplicated connector version found for {}: {} ({}) vs {} ({})", - c1.dockerRepository, c1.dockerImageTag, c1.connectorDefinitionId, c2.dockerImageTag, c2.connectorDefinitionId); + c1.dockerRepository, c1.dockerImageTag, c1.definitionId, c2.dockerImageTag, c2.definitionId); int comparison = v1.patchVersionCompareTo(v2); if (comparison >= 0) { return c1; diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/GoogleSecretsManager.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/GoogleSecretsManager.java new file mode 100644 index 000000000000..3ec017db15a0 --- /dev/null +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/GoogleSecretsManager.java @@ -0,0 +1,155 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.config.persistence; + +import com.google.api.gax.core.FixedCredentialsProvider; +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse; +import com.google.cloud.secretmanager.v1.ProjectName; +import com.google.cloud.secretmanager.v1.Replication; +import com.google.cloud.secretmanager.v1.Secret; +import com.google.cloud.secretmanager.v1.SecretManagerServiceClient; +import com.google.cloud.secretmanager.v1.SecretManagerServiceSettings; +import com.google.cloud.secretmanager.v1.SecretName; +import com.google.cloud.secretmanager.v1.SecretPayload; +import com.google.cloud.secretmanager.v1.SecretVersion; +import com.google.cloud.secretmanager.v1.SecretVersionName; +import com.google.common.base.Preconditions; +import com.google.protobuf.ByteString; +import io.airbyte.config.EnvConfigs; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Wrapper class to simplify the API for accessing secrets + */ +public class GoogleSecretsManager { + + /** + * Manual test fixture to make sure you've got your project id set in env and have appropriate creds + * to reach/write the secret store. + */ + public static void main(String[] args) throws Exception { + // Check that we're configured to a usable GCP project. + EnvConfigs envConfig = new EnvConfigs(); + String projectId = envConfig.getSecretStoreGcpProjectId(); + Preconditions.checkNotNull(projectId, "Project ID must not be empty"); + Preconditions.checkNotNull(Long.parseLong(projectId), "Project ID must be purely numeric, not %s".format(projectId)); + + // Check that we can read an existing one from that project / have permissions etc. + Preconditions.checkArgument(existsSecret("zzzzzz") == false, "Secret doesn't exist, should return false."); + Preconditions.checkArgument(existsSecret("dev_practice_sample_secret"), "Secret already exists, should return true."); + String content = readSecret("dev_practice_sample_secret"); + Preconditions.checkArgument("ThisIsMyTest".equals(content)); + + // Try creating a new one and reading it back. + String rand = UUID.randomUUID().toString(); + String key = "dev_practice_sample_" + rand; + saveSecret(key, rand); + String rand2 = readSecret(key); + Preconditions.checkArgument(rand.equals(rand2), "Values should have matched after writing and re-reading a new key."); + saveSecret(key, "foo"); + deleteSecret(key); + } + + public static String readSecret(String secretId) throws IOException { + EnvConfigs envConfig = new EnvConfigs(); + String projectId = envConfig.getSecretStoreGcpProjectId(); + try (SecretManagerServiceClient client = getSecretManagerServiceClient()) { + SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretId, "latest"); + AccessSecretVersionResponse response = client.accessSecretVersion(secretVersionName); + return response.getPayload().getData().toStringUtf8(); + } catch (com.google.api.gax.rpc.NotFoundException e) { + return null; + } + } + + private static SecretManagerServiceClient getSecretManagerServiceClient() throws IOException { + final ServiceAccountCredentials credentials = ServiceAccountCredentials + .fromStream(new ByteArrayInputStream((new EnvConfigs()).getSecretStoreGcpCredentials().getBytes(StandardCharsets.UTF_8))); + return SecretManagerServiceClient.create( + SecretManagerServiceSettings.newBuilder().setCredentialsProvider(FixedCredentialsProvider.create(credentials)).build()); + } + + public static boolean existsSecret(String secretId) throws IOException { + EnvConfigs envConfig = new EnvConfigs(); + String projectId = envConfig.getSecretStoreGcpProjectId(); + try (SecretManagerServiceClient client = getSecretManagerServiceClient()) { + System.out.println("Project ID: " + projectId); + System.out.println("Secret ID: " + secretId); + SecretVersionName secretVersionName = SecretVersionName.of(projectId, secretId, "latest"); + System.out.println(secretVersionName); + AccessSecretVersionResponse response = client.accessSecretVersion(secretVersionName); + return true; + } catch (com.google.api.gax.rpc.NotFoundException e) { + return false; + } + } + + public static void saveSecret(String secretId, String value) throws IOException { + EnvConfigs envConfig = new EnvConfigs(); + String projectId = envConfig.getSecretStoreGcpProjectId(); + try (SecretManagerServiceClient client = getSecretManagerServiceClient()) { + if (!existsSecret(secretId)) { + Secret secret = Secret.newBuilder().setReplication(Replication.newBuilder().setAutomatic( + Replication.Automatic.newBuilder().build()).build()).build(); + Secret createdSecret = client.createSecret(ProjectName.of(projectId), secretId, secret); + } + SecretPayload payload = SecretPayload.newBuilder() + .setData(ByteString.copyFromUtf8(value)) + .build(); + SecretVersion version = client.addSecretVersion(SecretName.of(projectId, secretId), payload); + } + } + + public static void deleteSecret(String secretId) throws IOException { + EnvConfigs envConfig = new EnvConfigs(); + String projectId = envConfig.getSecretStoreGcpProjectId(); + try (SecretManagerServiceClient client = getSecretManagerServiceClient()) { + SecretName secretName = SecretName.of(projectId, secretId); + client.deleteSecret(secretName); + } + } + + public static List listSecretsMatching(String prefix) throws IOException { + final String PREFIX_REGEX = "projects/\\d+/secrets/"; + List names = new ArrayList(); + try (SecretManagerServiceClient client = getSecretManagerServiceClient()) { + client.listSecrets(ProjectName.of(new EnvConfigs().getSecretStoreGcpProjectId())).iterateAll() + .forEach( + secret -> { + if (secret.getName().replaceFirst(PREFIX_REGEX, "").startsWith(prefix)) { + names.add(secret.getName().replaceFirst(PREFIX_REGEX, "")); + } + }); + } + return names; + } + +} diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/GoogleSecretsManagerConfigPersistence.java b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/GoogleSecretsManagerConfigPersistence.java new file mode 100644 index 000000000000..389babafce29 --- /dev/null +++ b/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/GoogleSecretsManagerConfigPersistence.java @@ -0,0 +1,165 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.config.persistence; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.AirbyteConfig; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.DestinationConnection; +import io.airbyte.config.SourceConnection; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +public class GoogleSecretsManagerConfigPersistence implements ConfigPersistence { + + public GoogleSecretsManagerConfigPersistence() {} + + public String getVersion() { + return "secrets-v1"; + } + + // @Override + public void loadData(ConfigPersistence seedPersistence) throws IOException { + loadData(seedPersistence, new HashSet()); + } + + public void loadData(ConfigPersistence seedPersistence, Set configsInUse) throws IOException { + // Don't need to do anything because the seed persistence only contains + // non-secret configs, which we don't load into the secrets store. + } + + /** + * Returns the definition ids for every connector we're storing. Hopefully this can be refactored + * into not existing once we have secrets as coordinates instead of storing the whole config as a + * single secret. + */ + public Set listDefinitionIdsInUseByConnectors() { + Set definitionIds = new HashSet(); + try { + List sources = listConfigs(ConfigSchema.SOURCE_CONNECTION, SourceConnection.class); + for (SourceConnection source : sources) { + definitionIds.add(source.getSourceDefinitionId().toString()); + } + List destinations = listConfigs(ConfigSchema.DESTINATION_CONNECTION, DestinationConnection.class); + for (DestinationConnection dest : destinations) { + definitionIds.add(dest.getDestinationDefinitionId().toString()); + } + return definitionIds; + } catch (IOException | JsonValidationException io) { + throw new RuntimeException(io); + } + } + + // @Override + public Set getRepositoriesFromDefinitionIds(Set usedConnectorDefinitionIds) throws IOException { + throw new UnsupportedOperationException( + "Secrets Manager does not store the list of definitions and thus cannot be used to look up docker repositories."); + } + + /** + * Determines the secrets manager key name for storing a particular config + */ + protected String generateKeyNameFromType(AirbyteConfig configType, String configId) { + return String.format("%s-%s-%s-configuration", getVersion(), configType.getIdFieldName(), configId); + } + + protected String generateKeyPrefixFromType(AirbyteConfig configType) { + return String.format("%s-%s-", getVersion(), configType.getIdFieldName()); + } + + @Override + public T getConfig(AirbyteConfig configType, String configId, Class clazz) + throws ConfigNotFoundException, JsonValidationException, IOException { + String keyName = generateKeyNameFromType(configType, configId); + return Jsons.deserialize(GoogleSecretsManager.readSecret(keyName), clazz); + } + + @Override + public List listConfigs(AirbyteConfig configType, Class clazz) throws JsonValidationException, IOException { + List configs = new ArrayList(); + for (String keyName : GoogleSecretsManager.listSecretsMatching(generateKeyPrefixFromType(configType))) { + configs.add(Jsons.deserialize(GoogleSecretsManager.readSecret(keyName), clazz)); + } + return configs; + } + + @Override + public void writeConfig(AirbyteConfig configType, String configId, T config) throws JsonValidationException, IOException { + String keyName = generateKeyNameFromType(configType, configId); + System.out.println("keyname " + keyName); + GoogleSecretsManager.saveSecret(keyName, Jsons.serialize(config)); + } + + @Override + public void deleteConfig(AirbyteConfig configType, String configId) throws ConfigNotFoundException, IOException { + String keyName = generateKeyNameFromType(configType, configId); + GoogleSecretsManager.deleteSecret(keyName); + } + + @Override + public void replaceAllConfigs(Map> configs, boolean dryRun) throws IOException { + if (dryRun) { + for (final Map.Entry> configuration : configs.entrySet()) { + configuration.getValue().forEach(Jsons::serialize); + } + return; + } + for (final Map.Entry> configuration : configs.entrySet()) { + AirbyteConfig configType = configuration.getKey(); + configuration.getValue().forEach(config -> { + try { + GoogleSecretsManager.saveSecret(generateKeyNameFromType(configType, configType.getId(config)), Jsons.serialize(config)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } + + @Override + public Map> dumpConfigs() throws IOException { + final Map> configs = new HashMap<>(); + + for (AirbyteConfig ctype : new ConfigSchema[] {ConfigSchema.SOURCE_CONNECTION, ConfigSchema.DESTINATION_CONNECTION}) { + List names = GoogleSecretsManager.listSecretsMatching(generateKeyPrefixFromType(ctype)); + final List configList = new ArrayList(); + for (String name : names) { + configList.add(Jsons.deserialize(GoogleSecretsManager.readSecret(name), JsonNode.class)); + } + configs.put(ctype.name(), configList.stream()); + } + + return configs; + } + +} diff --git a/airbyte-config/persistence/src/test-integration/java/io/airbyte/config/persistence/GoogleSecretsManagerConfigPersistenceTest.java b/airbyte-config/persistence/src/test-integration/java/io/airbyte/config/persistence/GoogleSecretsManagerConfigPersistenceTest.java new file mode 100644 index 000000000000..bb8bcdc3876e --- /dev/null +++ b/airbyte-config/persistence/src/test-integration/java/io/airbyte/config/persistence/GoogleSecretsManagerConfigPersistenceTest.java @@ -0,0 +1,101 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.config.persistence; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.google.common.collect.Sets; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.StandardDestinationDefinition; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.util.UUID; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class GoogleSecretsManagerConfigPersistenceTest { + + private static final UUID WORKSPACE_ID = UUID.randomUUID(); + + private GoogleSecretsManagerConfigPersistence configPersistence; + + public static final UUID UUID_1 = new UUID(0, 1); + public static final StandardSourceDefinition SOURCE_1 = new StandardSourceDefinition(); + static { + SOURCE_1.withSourceDefinitionId(UUID_1).withName("postgresql"); + } + + public static final UUID UUID_2 = new UUID(0, 2); + public static final StandardSourceDefinition SOURCE_2 = new StandardSourceDefinition(); + static { + SOURCE_2.withSourceDefinitionId(UUID_2).withName("apache storm"); + } + + @BeforeEach + void setUp() throws IOException { + configPersistence = new GoogleSecretsManagerConfigPersistence(); + } + + @Test + void testReadWriteConfig() throws IOException, JsonValidationException, ConfigNotFoundException { + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, UUID_1.toString(), SOURCE_1); + Assertions.assertEquals(SOURCE_1, + configPersistence.getConfig( + ConfigSchema.STANDARD_SOURCE_DEFINITION, + UUID_1.toString(), + StandardSourceDefinition.class)); + } + + @Test + void testListConfigs() throws JsonValidationException, IOException { + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, UUID_1.toString(), SOURCE_1); + configPersistence.writeConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, UUID_2.toString(), SOURCE_2); + + Assertions.assertEquals( + Sets.newHashSet(SOURCE_1, SOURCE_2), + Sets.newHashSet(configPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class))); + } + + private void assertRecordCount(int expectedCount) throws Exception { + // Result> recordCount = database.query(ctx -> + // ctx.select(count(asterisk())).from(table("airbyte_configs")).fetch()); + assertEquals(expectedCount, 999);// TODO: Fix // recordCount.get(0).value1()); + } + + private void assertHasSource(StandardSourceDefinition source) throws Exception { + Assertions.assertEquals(source, configPersistence + .getConfig(ConfigSchema.STANDARD_SOURCE_DEFINITION, source.getSourceDefinitionId().toString(), + StandardSourceDefinition.class)); + } + + private void assertHasDestination(StandardDestinationDefinition destination) throws Exception { + Assertions.assertEquals(destination, configPersistence + .getConfig(ConfigSchema.STANDARD_DESTINATION_DEFINITION, destination.getDestinationDefinitionId().toString(), + StandardDestinationDefinition.class)); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java index 1987729698e4..acbd68d04cb2 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceLoadDataTest.java @@ -26,16 +26,13 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; @@ -43,9 +40,7 @@ import io.airbyte.config.StandardSourceDefinition; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import java.util.Collections; -import java.util.Map; import java.util.UUID; -import java.util.stream.Stream; import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -84,26 +79,10 @@ public void resetPersistence() { @Test @Order(1) - @DisplayName("When database is empty, seed should be copied to the database") - public void testCopyConfigsToEmptyDatabase() throws Exception { - Map> initialSeeds = Map.of( - ConfigSchema.STANDARD_DESTINATION_DEFINITION.name(), Stream.of(Jsons.jsonNode(DESTINATION_SNOWFLAKE)), - ConfigSchema.STANDARD_SOURCE_DEFINITION.name(), Stream.of(Jsons.jsonNode(SOURCE_GITHUB))); - when(seedPersistence.dumpConfigs()).thenReturn(initialSeeds); - - configPersistence.loadData(seedPersistence); - assertRecordCount(2); - assertHasSource(SOURCE_GITHUB); - assertHasDestination(DESTINATION_SNOWFLAKE); - verify(configPersistence, times(1)).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); - verify(configPersistence, never()).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); - } - - @Test - @Order(2) - @DisplayName("When database is not empty, configs should be updated") + @DisplayName("When database is empty, configs should be inserted") public void testUpdateConfigsInNonEmptyDatabase() throws Exception { - // the seed has two destinations, one of which (S3) is new + when(seedPersistence.listConfigs(ConfigSchema.STANDARD_SOURCE_DEFINITION, StandardSourceDefinition.class)) + .thenReturn(Lists.newArrayList(SOURCE_GITHUB)); when(seedPersistence.listConfigs(ConfigSchema.STANDARD_DESTINATION_DEFINITION, StandardDestinationDefinition.class)) .thenReturn(Lists.newArrayList(DESTINATION_S3, DESTINATION_SNOWFLAKE)); @@ -113,12 +92,11 @@ public void testUpdateConfigsInNonEmptyDatabase() throws Exception { assertRecordCount(3); assertHasDestination(DESTINATION_SNOWFLAKE); - verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); verify(configPersistence, times(1)).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); } @Test - @Order(3) + @Order(2) @DisplayName("When a connector is in use, its definition should not be updated") public void testNoUpdateForUsedConnector() throws Exception { // the seed has a newer version of s3 destination and github source @@ -150,7 +128,7 @@ public void testNoUpdateForUsedConnector() throws Exception { } @Test - @Order(4) + @Order(3) @DisplayName("When a connector is not in use, its definition should be updated") public void testUpdateForUnusedConnector() throws Exception { // the seed has a newer version of snowflake destination diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java new file mode 100644 index 000000000000..9e4faebb8cbd --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceMigrateFileConfigsTest.java @@ -0,0 +1,128 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.config.persistence; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import io.airbyte.config.Configs; +import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.jooq.DSLContext; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Unit test for the {@link DatabaseConfigPersistence#migrateFileConfigs} method. + */ +public class DatabaseConfigPersistenceMigrateFileConfigsTest extends BaseDatabaseConfigPersistenceTest { + + private static Path ROOT_PATH; + private final Configs configs = mock(Configs.class); + + @BeforeAll + public static void setup() throws Exception { + database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + configPersistence = spy(new DatabaseConfigPersistence(database)); + } + + @AfterAll + public static void tearDown() throws Exception { + database.close(); + } + + @BeforeEach + public void resetPersistence() throws Exception { + ROOT_PATH = Files.createTempDirectory( + Files.createDirectories(Path.of("/tmp/airbyte_tests")), + DatabaseConfigPersistenceMigrateFileConfigsTest.class.getSimpleName() + UUID.randomUUID()); + + reset(configs); + when(configs.getConfigRoot()).thenReturn(ROOT_PATH); + + database.query(ctx -> ctx.truncateTable("airbyte_configs").execute()); + + reset(configPersistence); + } + + @Test + @DisplayName("When database is not initialized, and there is no local config dir, do nothing") + public void testNewDeployment() throws Exception { + configPersistence.migrateFileConfigs(configs); + + assertRecordCount(0); + + verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + verify(configPersistence, never()).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + @Test + @DisplayName("When database is not initialized, and there is local config dir, copy from local dir") + public void testMigrationDeployment() throws Exception { + prepareLocalFilePersistence(); + + configPersistence.migrateFileConfigs(configs); + + assertRecordCount(2); + assertHasSource(SOURCE_GITHUB); + assertHasDestination(DESTINATION_S3); + + verify(configPersistence, times(1)).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + @Test + @DisplayName("When database has been initialized, do nothing") + public void testUpdateDeployment() throws Exception { + prepareLocalFilePersistence(); + writeSource(configPersistence, SOURCE_GITHUB); + + configPersistence.migrateFileConfigs(configs); + + assertRecordCount(1); + assertHasSource(SOURCE_GITHUB); + + verify(configPersistence, never()).copyConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + verify(configPersistence, never()).updateConfigsFromSeed(any(DSLContext.class), any(ConfigPersistence.class)); + } + + private void prepareLocalFilePersistence() throws Exception { + Files.createDirectories(ROOT_PATH.resolve(FileSystemConfigPersistence.CONFIG_DIR)); + ConfigPersistence filePersistence = new FileSystemConfigPersistence(ROOT_PATH); + writeSource(filePersistence, SOURCE_GITHUB); + writeDestination(filePersistence, DESTINATION_S3); + } + +} diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java index f32a9c3156f1..91b61ef2a5e6 100644 --- a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceTest.java @@ -25,7 +25,9 @@ package io.airbyte.config.persistence; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.spy; import com.fasterxml.jackson.databind.JsonNode; @@ -37,8 +39,10 @@ import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; import java.time.OffsetDateTime; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; @@ -46,8 +50,9 @@ import org.junit.jupiter.api.Test; /** - * The {@link DatabaseConfigPersistence#loadData} method is tested in - * {@link DatabaseConfigPersistenceLoadDataTest}. + * See {@link DatabaseConfigPersistenceLoadDataTest}, + * {@link DatabaseConfigPersistenceMigrateFileConfigsTest}, and + * {@link DatabaseConfigPersistenceUpdateConnectorDefinitionsTest} for testing of specific methods. */ public class DatabaseConfigPersistenceTest extends BaseDatabaseConfigPersistenceTest { @@ -218,4 +223,31 @@ public void testUpdateConfigRecord() throws Exception { assertHasSource(SOURCE_GITHUB); } + @Test + public void testHasNewVersion() { + assertTrue(DatabaseConfigPersistence.hasNewVersion("0.1.99", "0.2.0")); + assertFalse(DatabaseConfigPersistence.hasNewVersion("invalid_version", "0.2.0")); + } + + @Test + public void testGetNewFields() { + JsonNode o1 = Jsons.deserialize("{ \"field1\": 1, \"field2\": 2 }"); + JsonNode o2 = Jsons.deserialize("{ \"field1\": 1, \"field3\": 3 }"); + assertEquals(Collections.emptySet(), DatabaseConfigPersistence.getNewFields(o1, o1)); + assertEquals(Collections.singleton("field3"), DatabaseConfigPersistence.getNewFields(o1, o2)); + assertEquals(Collections.singleton("field2"), DatabaseConfigPersistence.getNewFields(o2, o1)); + } + + @Test + public void testGetDefinitionWithNewFields() { + JsonNode current = Jsons.deserialize("{ \"field1\": 1, \"field2\": 2 }"); + JsonNode latest = Jsons.deserialize("{ \"field1\": 1, \"field3\": 3, \"field4\": 4 }"); + Set newFields = Set.of("field3"); + + assertEquals(current, DatabaseConfigPersistence.getDefinitionWithNewFields(current, latest, Collections.emptySet())); + + JsonNode currentWithNewFields = Jsons.deserialize("{ \"field1\": 1, \"field2\": 2, \"field3\": 3 }"); + assertEquals(currentWithNewFields, DatabaseConfigPersistence.getDefinitionWithNewFields(current, latest, newFields)); + } + } diff --git a/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java new file mode 100644 index 000000000000..bbf7752c8c8e --- /dev/null +++ b/airbyte-config/persistence/src/test/java/io/airbyte/config/persistence/DatabaseConfigPersistenceUpdateConnectorDefinitionsTest.java @@ -0,0 +1,190 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.config.persistence; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.ConfigSchema; +import io.airbyte.config.StandardSourceDefinition; +import io.airbyte.config.persistence.DatabaseConfigPersistence.ConnectorInfo; +import io.airbyte.db.instance.configs.ConfigsDatabaseInstance; +import java.io.IOException; +import java.sql.SQLException; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Unit test for the {@link DatabaseConfigPersistence#updateConnectorDefinitions} method. + */ +public class DatabaseConfigPersistenceUpdateConnectorDefinitionsTest extends BaseDatabaseConfigPersistenceTest { + + private static final JsonNode SOURCE_GITHUB_JSON = Jsons.jsonNode(SOURCE_GITHUB); + private static final OffsetDateTime TIMESTAMP = OffsetDateTime.now(); + + @BeforeAll + public static void setup() throws Exception { + database = new ConfigsDatabaseInstance(container.getUsername(), container.getPassword(), container.getJdbcUrl()).getAndInitialize(); + configPersistence = new DatabaseConfigPersistence(database); + } + + @AfterAll + public static void tearDown() throws Exception { + database.close(); + } + + @BeforeEach + public void resetDatabase() throws SQLException { + database.transaction(ctx -> ctx.truncateTable("airbyte_configs").execute()); + } + + @Test + @DisplayName("When a connector does not exist, add it") + public void testNewConnector() throws Exception { + assertUpdateConnectorDefinition( + Collections.emptyList(), + Collections.emptyList(), + List.of(SOURCE_GITHUB), + Collections.singletonList(SOURCE_GITHUB)); + } + + @Test + @DisplayName("When an old connector is in use, if it has all fields, do not update it") + public void testOldConnectorInUseWithAllFields() throws Exception { + StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.0.0"); + StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.1000.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.singletonList(currentSource), + Collections.singletonList(latestSource), + Collections.singletonList(currentSource)); + } + + @Test + @DisplayName("When a old connector is in use, add missing fields, do not update its version") + public void testOldConnectorInUseWithMissingFields() throws Exception { + StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.0.0").withDocumentationUrl(null).withSourceType(null); + StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.1000.0"); + StandardSourceDefinition currentSourceWithNewFields = getSource().withDockerImageTag("0.0.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.singletonList(currentSource), + Collections.singletonList(latestSource), + Collections.singletonList(currentSourceWithNewFields)); + } + + @Test + @DisplayName("When an unused connector has a new version, update it") + public void testUnusedConnectorWithOldVersion() throws Exception { + StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.0.0"); + StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.1000.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.emptyList(), + Collections.singletonList(latestSource), + Collections.singletonList(latestSource)); + } + + @Test + @DisplayName("When an unused connector has missing fields, add the missing fields, do not update its version") + public void testUnusedConnectorWithMissingFields() throws Exception { + StandardSourceDefinition currentSource = getSource().withDockerImageTag("0.1000.0").withDocumentationUrl(null).withSourceType(null); + StandardSourceDefinition latestSource = getSource().withDockerImageTag("0.99.0"); + StandardSourceDefinition currentSourceWithNewFields = getSource().withDockerImageTag("0.1000.0"); + + assertUpdateConnectorDefinition( + Collections.singletonList(currentSource), + Collections.emptyList(), + Collections.singletonList(latestSource), + Collections.singletonList(currentSourceWithNewFields)); + } + + /** + * Clone a source for modification and testing. + */ + private StandardSourceDefinition getSource() { + return Jsons.object(Jsons.clone(SOURCE_GITHUB_JSON), StandardSourceDefinition.class); + } + + /** + * @param currentSources all sources currently exist in the database + * @param currentSourcesInUse a subset of currentSources; sources currently used in data syncing + */ + private void assertUpdateConnectorDefinition(List currentSources, + List currentSourcesInUse, + List latestSources, + List expectedUpdatedSources) + throws Exception { + for (StandardSourceDefinition source : currentSources) { + writeSource(configPersistence, source); + } + + for (StandardSourceDefinition source : currentSourcesInUse) { + assertTrue(currentSources.contains(source), "currentSourcesInUse must exist in currentSources"); + } + + Set sourceRepositoriesInUse = currentSourcesInUse.stream() + .map(StandardSourceDefinition::getDockerRepository) + .collect(Collectors.toSet()); + Map currentSourceRepositoryToInfo = currentSources.stream() + .collect(Collectors.toMap( + StandardSourceDefinition::getDockerRepository, + s -> new ConnectorInfo(s.getSourceDefinitionId().toString(), Jsons.jsonNode(s)))); + + database.transaction(ctx -> { + try { + configPersistence.updateConnectorDefinitions( + ctx, + TIMESTAMP, + ConfigSchema.STANDARD_SOURCE_DEFINITION, + latestSources, + sourceRepositoriesInUse, + currentSourceRepositoryToInfo); + } catch (IOException e) { + throw new SQLException(e); + } + return null; + }); + + assertRecordCount(expectedUpdatedSources.size()); + for (StandardSourceDefinition source : expectedUpdatedSources) { + assertHasSource(source); + } + } + +} diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java b/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java index a18ef87b95b4..65e17a8c9fdc 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java @@ -108,9 +108,18 @@ private void setJsonField(Field field, FieldValue fieldValue, ObjectNode node) { } } else if (fieldValue.getAttribute().equals(Attribute.RECORD)) { ObjectNode newNode = node.putObject(fieldName); - field.getSubFields().forEach(recordField -> { - setJsonField(recordField, fieldValue.getRecordValue().get(recordField.getName()), newNode); - }); + FieldList subFields = field.getSubFields(); + try { + // named get doesn't work here with nested arrays and objects; index is the only correlation between + // field and field value + if (subFields != null && !subFields.isEmpty()) { + for (int i = 0; i < subFields.size(); i++) { + setJsonField(field.getSubFields().get(i), fieldValue.getRecordValue().get(i), newNode); + } + } + } catch (UnsupportedOperationException e) { + LOGGER.error("Failed to parse Object field with name: ", fieldName, e.getMessage()); + } } } diff --git a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_29_15_001__Add_temporalWorkflowId_col_to_Attempts.java b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_29_15_001__Add_temporalWorkflowId_col_to_Attempts.java index 5cc5ef8828c6..10fee8d05d13 100644 --- a/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_29_15_001__Add_temporalWorkflowId_col_to_Attempts.java +++ b/airbyte-db/lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_29_15_001__Add_temporalWorkflowId_col_to_Attempts.java @@ -38,7 +38,8 @@ public void migrate(Context context) throws Exception { // As database schema changes, the generated jOOQ code can be deprecated. So // old migration may not compile if there is any generated code. DSLContext ctx = DSL.using(context.getConnection()); - ctx.alterTable("attempts").addColumn(DSL.field("temporal_workflow_id", SQLDataType.VARCHAR(256).nullable(true))) + ctx.alterTable("attempts") + .addColumnIfNotExists(DSL.field("temporal_workflow_id", SQLDataType.VARCHAR(256).nullable(true))) .execute(); } diff --git a/airbyte-db/lib/src/main/resources/jobs_database/Attempts.yaml b/airbyte-db/lib/src/main/resources/jobs_database/Attempts.yaml index dea43213d71d..758f53c322f6 100644 --- a/airbyte-db/lib/src/main/resources/jobs_database/Attempts.yaml +++ b/airbyte-db/lib/src/main/resources/jobs_database/Attempts.yaml @@ -11,7 +11,7 @@ required: - status - created_at - updated_at -additionalProperties: false +additionalProperties: true properties: id: type: number @@ -25,8 +25,6 @@ properties: type: ["null", object] status: type: string - temporal_workflow_id: - type: ["null", string] created_at: # todo should be datetime. type: string diff --git a/airbyte-e2e-testing/cypress/integration/onboarding.spec.js b/airbyte-e2e-testing/cypress/integration/onboarding.spec.js index 15a6a869d720..4c1c14b9ae38 100644 --- a/airbyte-e2e-testing/cypress/integration/onboarding.spec.js +++ b/airbyte-e2e-testing/cypress/integration/onboarding.spec.js @@ -8,9 +8,6 @@ describe("Onboarding actions", () => { cy.submit(); - cy.url().should("include", `${Cypress.config().baseUrl}/onboarding`); - cy.get("button[data-id='skip-onboarding']").click(); - cy.url().should("equal", `${Cypress.config().baseUrl}/`); }); }); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md index fc347b985bda..df8a7fe01876 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md +++ b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md @@ -29,7 +29,9 @@ Our SSH connector support is designed to be easy to plug into any existing conne config = TransformConfig.get_ssh_altered_config(config, port_key="port", host_key="host") ``` Replace port_key and host_key as necessary. Look at `transform_postgres()` to see an example. -2. If your `host_key="host"` and `port_key="port"` then step 1 should be sufficient. However if the key names differ for your connector, you will also need to add some logic into `sshtunneling.sh` (within airbyte-workers) to handle this, as currently it assumes that the keys are exactly `host` and `port`. +2. To make sure your changes are present in Normalization when running tests on the connector locally, you'll need to change [this version tag](https://github.com/airbytehq/airbyte/blob/6d9ba022646441c7f298ca4dcaa3df59b9a19fbb/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java#L50) to `dev` so that the new locally built docker image for Normalization is used. Don't push this change with the PR though. +3. If your `host_key="host"` and `port_key="port"` then this step is not necessary. However if the key names differ for your connector, you will also need to add some logic into `sshtunneling.sh` (within airbyte-workers) to handle this, as currently it assumes that the keys are exactly `host` and `port`. +4. When making your PR, make sure that you've version bumped Normalization (in `airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java` and `airbyte-integrations/bases/base-normalization/Dockerfile`). You'll need to /test & /publish Normalization _first_ so that when you /test the connector, it can use the new version. ## Misc diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 314ef47ef073..3dba80b4e92f 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -45,5 +45,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.1.44 +LABEL io.airbyte.version=0.1.45 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 9df5d98b497b..df65bbeadd03 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -48,6 +48,7 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs } integrationTest.dependsOn("customIntegrationTestPython") +customIntegrationTests.dependsOn("customIntegrationTestPython") // TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead task('mypyCheck', type: PythonTask) { diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index 91b27741decf..3157369bb6a6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -169,7 +169,7 @@ def generate_profile_yaml_file(self, destination_type: DestinationType, test_roo profiles_config = config_generator.read_json_config(f"../secrets/{destination_type.value.lower()}.json") # Adapt credential file to look like destination config.json if destination_type.value == DestinationType.BIGQUERY.value: - credentials = profiles_config + credentials = profiles_config["basic_bigquery_config"] profiles_config = { "credentials_json": json.dumps(credentials), "dataset_id": self.target_schema, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 72c750f604ca..0f08a0786347 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -30,9 +30,14 @@ select cast(DATE as date ) as DATE, - cast(TIMESTAMP_COL as - timestamp with time zone -) as TIMESTAMP_COL, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , cast("HKD@spéçiäl & characters" as float ) as "HKD@spéçiäl & characters", diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql index f78fa56bbf26..5343c1b11041 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -30,9 +30,14 @@ select cast(DATE as date ) as DATE, - cast(TIMESTAMP_COL as - timestamp with time zone -) as TIMESTAMP_COL, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , cast("HKD@spéçiäl & characters" as float ) as "HKD@spéçiäl & characters", diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql index cd3d7238d5fd..a076fd62c64a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -4,7 +4,14 @@ select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, cast(DATE as {{ type_date() }}) as DATE, - cast(TIMESTAMP_COL as {{ type_timestamp_with_timezone() }}) as TIMESTAMP_COL, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql index 49213c4c56d1..812cf2578aa4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_primary_key_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -4,7 +4,14 @@ select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, cast(DATE as {{ type_date() }}) as DATE, - cast(TIMESTAMP_COL as {{ type_timestamp_with_timezone() }}) as TIMESTAMP_COL, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/messages.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/messages.txt index 5749d5ec3476..a5ee59e60d67 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/messages.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/messages.txt @@ -1,17 +1,17 @@ -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637589000, "data": { "id": 1, "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 2.13, "HKD_special___characters": "column name collision?" }}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637689100, "data": { "id": 1, "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 7.15, "HKD_special___characters": "column name collision?"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637789200, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00Z", "NZD": 3.89, "HKD@spéçiäl & characters": 7.12, "HKD_special___characters": "column name collision?", "USD": 10.16}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637889300, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 7.99, "HKD_special___characters": "column name collision?", "USD": 10.99}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637589000, "data": { "id": 1, "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "NZD": 1.14, "HKD@spéçiäl & characters": 2.13, "HKD_special___characters": "column name collision?" }}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637689100, "data": { "id": 1, "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "NZD": 1.14, "HKD@spéçiäl & characters": 7.15, "HKD_special___characters": "column name collision?"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637789200, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "NZD": 3.89, "HKD@spéçiäl & characters": 7.12, "HKD_special___characters": "column name collision?", "USD": 10.16}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637889300, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+0000", "NZD": 1.14, "HKD@spéçiäl & characters": 7.99, "HKD_special___characters": "column name collision?", "USD": 10.99}}} {"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637989400, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 8, "HKD_special___characters": "column name collision?", "USD": 10.16}}} {"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990700, "data": { "id": 1, "currency": "USD", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 10.5, "HKD_special___characters": "column name collision?"}}} {"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?"}}} {"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990900, "data": { "id": 3, "currency": "GBP", "NZD": 3.14, "HKD@spéçiäl & characters": 9.2, "HKD_special___characters": "column name collision?"}}} {"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991000, "data": { "id": 2, "currency": "EUR", "NZD": 3.89, "HKD@spéçiäl & characters": 7.02, "HKD_special___characters": "column name collision?"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637589000, "data": { "id": 1, "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 2.13, "HKD_special___characters": "column name collision?"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637689100, "data": { "id": 1, "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 7.15, "HKD_special___characters": "column name collision?"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637789200, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00Z", "NZD": 3.89, "HKD@spéçiäl & characters": 7.12, "HKD_special___characters": "column name collision?", "USD": 10.16}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637889300, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 7.99, "HKD_special___characters": "column name collision?", "USD": 10.99}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637589000, "data": { "id": 1, "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "NZD": 1.14, "HKD@spéçiäl & characters": 2.13, "HKD_special___characters": "column name collision?"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637689100, "data": { "id": 1, "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "NZD": 1.14, "HKD@spéçiäl & characters": 7.15, "HKD_special___characters": "column name collision?"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637789200, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "NZD": 3.89, "HKD@spéçiäl & characters": 7.12, "HKD_special___characters": "column name collision?", "USD": 10.16}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637889300, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+0000", "NZD": 1.14, "HKD@spéçiäl & characters": 7.99, "HKD_special___characters": "column name collision?", "USD": 10.99}}} {"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637989400, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 8, "HKD_special___characters": "column name collision?", "USD": 10.16}}} {"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990700, "data": { "id": 1, "currency": "USD", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 10.5, "HKD_special___characters": "column name collision?"}}} {"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?"}}} diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 61b222b12a95..65f7a0c878d0 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -435,7 +435,12 @@ def cast_property_type(self, property_name: str, column_name: str, jinja_column: elif is_number(definition["type"]): sql_type = jinja_call("dbt_utils.type_float()") elif is_timestamp_with_time_zone(definition): + if self.destination_type == DestinationType.SNOWFLAKE: + # snowflake uses case when statement to parse timestamp field + # in this case [cast] operator is not needed as data already converted to timestamp type + return self.generate_snowflake_timestamp_statement(column_name) sql_type = jinja_call("type_timestamp_with_timezone()") + elif is_date(definition): sql_type = jinja_call("type_date()") elif is_string(definition["type"]): @@ -446,6 +451,31 @@ def cast_property_type(self, property_name: str, column_name: str, jinja_column: return f"cast({column_name} as {sql_type}) as {column_name}" + def generate_snowflake_timestamp_statement(self, column_name: str) -> str: + """ + Generates snowflake DB specific timestamp case when statement + """ + formats = [ + {"regex": r"\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}", "format": "YYYY-MM-DDTHH24:MI:SSTZHTZM"}, + {"regex": r"\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}", "format": "YYYY-MM-DDTHH24:MI:SSTZH"}, + { + "regex": r"\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}", + "format": "YYYY-MM-DDTHH24:MI:SS.FFTZHTZM", + }, + {"regex": r"\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}", "format": "YYYY-MM-DDTHH24:MI:SS.FFTZH"}, + ] + template = Template( + """ + case + {% for format_item in formats %} + when {{column_name}} regexp '{{format_item['regex']}}' then to_timestamp_tz({{column_name}}, '{{format_item['format']}}') + {% endfor %} + else to_timestamp_tz({{column_name}}) + end as {{column_name}} + """ + ) + return template.render(formats=formats, column_name=column_name) + def generate_id_hashing_model(self, from_table: str, column_names: Dict[str, Tuple[str, str]]) -> str: template = Template( diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py index 33a6d2183e94..dd9c7439344c 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py @@ -145,7 +145,7 @@ def get_top_level_item(variant_path: List[str]): """ variant_props = [set(list(v["properties"].keys())) for v in variants] common_props = set.intersection(*variant_props) - assert common_props, "There should be at least one common property for oneOf subojects" + assert common_props, "There should be at least one common property for oneOf subobjects" assert any( [all(["const" in var["properties"][prop] for var in variants]) for prop in common_props] ), f"Any of {common_props} properties in {'.'.join(variant_path)} has no const keyword. See specification reference at https://docs.airbyte.io/connector-development/connector-specification-reference" diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java index ac252c683259..2941f0e0e62b 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -56,6 +56,8 @@ import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; import io.airbyte.protocol.models.SyncMode; import io.airbyte.workers.DbtTransformationRunner; import io.airbyte.workers.DefaultCheckConnectionWorker; @@ -82,9 +84,12 @@ import java.util.Map.Entry; import java.util.Random; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import org.joda.time.DateTime; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; @@ -383,10 +388,9 @@ public void testSyncWithLargeRecordBatch(String messagesFilename, String catalog final List messages = MoreResources.readResource(messagesFilename).lines() .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); - final List largeNumberRecords = Collections.nCopies(1000, messages).stream().flatMap(List::stream).collect(Collectors.toList()); + final List largeNumberRecords = Collections.nCopies(400, messages).stream().flatMap(List::stream).collect(Collectors.toList()); final JsonNode config = getConfig(); - final String defaultSchema = getDefaultSchema(config); runSyncAndVerifyStateOutput(config, largeNumberRecords, configuredCatalog, false); } @@ -1156,4 +1160,125 @@ public String toString() { } + /** + * This test MUST be disabled by default, but you may uncomment it and use when need to reproduce a + * performance issue for destination. This test helps you to emulate lot's of stream and messages in + * each simply changing the "streamsSize" args to set a number of tables\streams and the + * "messagesNumber" to a messages number that would be written in each stream. !!! Do NOT forget to + * manually remove all generated objects !!! Hint: To check the destination container output run + * "docker ps" command in console to find the container's id. Then run "docker container attach + * your_containers_id" (ex. docker container attach 18cc929f44c8) to see the container's output + */ + @Test + @Disabled + public void testStressPerformance() throws Exception { + final int streamsSize = 5; // number of generated streams + final int messagesNumber = 300; // number of msg to be written to each generated stream + + // Each stream will have an id and name fields + final String USERS_STREAM_NAME = "users"; // stream's name prefix. Will get "user0", "user1", etc. + final String ID = "id"; + final String NAME = "name"; + + // generate schema\catalogs + List configuredAirbyteStreams = new ArrayList<>(); + for (int i = 0; i < streamsSize; i++) { + configuredAirbyteStreams + .add(CatalogHelpers.createAirbyteStream(USERS_STREAM_NAME + i, + Field.of(NAME, JsonSchemaPrimitive.STRING), + Field + .of(ID, JsonSchemaPrimitive.STRING))); + } + final AirbyteCatalog testCatalog = new AirbyteCatalog().withStreams(configuredAirbyteStreams); + final ConfiguredAirbyteCatalog configuredTestCatalog = CatalogHelpers + .toDefaultConfiguredCatalog(testCatalog); + + final JsonNode config = getConfig(); + final WorkerDestinationConfig destinationConfig = new WorkerDestinationConfig() + .withConnectionId(UUID.randomUUID()) + .withCatalog(configuredTestCatalog) + .withDestinationConnectionConfiguration(config); + final AirbyteDestination destination = getDestination(); + + // Start destination + destination.start(destinationConfig, jobRoot); + + AtomicInteger currentStreamNumber = new AtomicInteger(0); + AtomicInteger currentRecordNumberForStream = new AtomicInteger(0); + + // this is just a current state logger. Useful when running long hours tests to see the progress + Thread countPrinter = new Thread(() -> { + while (true) { + System.out.println( + "currentStreamNumber=" + currentStreamNumber + ", currentRecordNumberForStream=" + + currentRecordNumberForStream + ", " + DateTime.now()); + try { + Thread.sleep(10000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + }); + countPrinter.start(); + + // iterate through streams + for (int streamCounter = 0; streamCounter < streamsSize; streamCounter++) { + LOGGER.info("Started new stream processing with #" + streamCounter); + // iterate through msm inside a particular stream + // Generate messages and put it to stream + for (int msgCounter = 0; msgCounter < messagesNumber; msgCounter++) { + AirbyteMessage msg = new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(USERS_STREAM_NAME + streamCounter) + .withData( + Jsons.jsonNode( + ImmutableMap.builder().put(NAME, LOREM_IPSUM) + .put(ID, streamCounter + "_" + msgCounter) + .build())) + .withEmittedAt(Instant.now().toEpochMilli())); + try { + destination.accept(msg); + } catch (Exception e) { + LOGGER.error("Failed to write a RECORD message: " + e); + throw new RuntimeException(e); + } + + currentRecordNumberForStream.set(msgCounter); + } + + // send state message here, it's required + AirbyteMessage msgState = new AirbyteMessage() + .withType(AirbyteMessage.Type.STATE) + .withState(new AirbyteStateMessage() + .withData( + Jsons.jsonNode(ImmutableMap.builder().put("start_date", "2020-09-02").build()))); + try { + destination.accept(msgState); + } catch (Exception e) { + LOGGER.error("Failed to write a STATE message: " + e); + throw new RuntimeException(e); + } + + currentStreamNumber.set(streamCounter); + } + + LOGGER.info(String + .format("Added %s messages to each of %s streams", currentRecordNumberForStream, + currentStreamNumber)); + // Close destination + destination.notifyEndOfStream(); + } + + private final static String LOREM_IPSUM = + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque malesuada lacinia aliquet. Nam feugiat mauris vel magna dignissim feugiat. Nam non dapibus sapien, ac mattis purus. Donec mollis libero erat, a rutrum ipsum pretium id. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Integer nec aliquam leo. Aliquam eu dictum augue, a ornare elit.\n" + + "\n" + + "Nulla viverra blandit neque. Nam blandit varius efficitur. Nunc at sapien blandit, malesuada lectus vel, tincidunt orci. Proin blandit metus eget libero facilisis interdum. Aenean luctus scelerisque orci, at scelerisque sem vestibulum in. Nullam ornare massa sed dui efficitur, eget volutpat lectus elementum. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Integer elementum mi vitae erat eleifend iaculis. Nullam eget tincidunt est, eget tempor est. Sed risus velit, iaculis vitae est in, volutpat consectetur odio. Aenean ut fringilla elit. Suspendisse non aliquet massa. Curabitur suscipit metus nunc, nec porttitor velit venenatis vel. Fusce vestibulum eleifend diam, lobortis auctor magna.\n" + + "\n" + + "Etiam maximus, mi feugiat pharetra mattis, nulla neque euismod metus, in congue nunc sem nec ligula. Curabitur aliquam, risus id convallis cursus, nunc orci sollicitudin enim, quis scelerisque nibh dui in ipsum. Suspendisse mollis, metus a dapibus scelerisque, sapien nulla pretium ipsum, non finibus sem orci et lectus. Aliquam dictum magna nisi, a consectetur urna euismod nec. In pulvinar facilisis nulla, id mollis libero pulvinar vel. Nam a commodo leo, eu commodo dolor. In hac habitasse platea dictumst. Curabitur auctor purus quis tortor laoreet efficitur. Quisque tincidunt, risus vel rutrum fermentum, libero urna dignissim augue, eget pulvinar nibh ligula ut tortor. Vivamus convallis non risus sed consectetur. Etiam accumsan enim ac nisl suscipit, vel congue lorem volutpat. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce non orci quis lacus rhoncus vestibulum nec ut magna. In varius lectus nec quam posuere finibus. Vivamus quis lectus vitae tortor sollicitudin fermentum.\n" + + "\n" + + "Pellentesque elementum vehicula egestas. Sed volutpat velit arcu, at imperdiet sapien consectetur facilisis. Suspendisse porttitor tincidunt interdum. Morbi gravida faucibus tortor, ut rutrum magna tincidunt a. Morbi eu nisi eget dui finibus hendrerit sit amet in augue. Aenean imperdiet lacus enim, a volutpat nulla placerat at. Suspendisse nibh ipsum, venenatis vel maximus ut, fringilla nec felis. Sed risus mi, egestas quis quam ullamcorper, pharetra vestibulum diam.\n" + + "\n" + + "Praesent finibus scelerisque elit, accumsan condimentum risus mattis vitae. Donec tristique hendrerit facilisis. Curabitur metus purus, venenatis non elementum id, finibus eu augue. Quisque posuere rhoncus ligula, et vehicula erat pulvinar at. Pellentesque vel quam vel lectus tincidunt congue quis id sapien. Ut efficitur mauris vitae pretium iaculis. Aliquam consectetur iaculis nisi vitae laoreet. Integer vel odio quis diam mattis tempor eget nec est. Donec iaculis facilisis neque, at dictum magna vestibulum ut. Sed malesuada non nunc ac consequat. Maecenas tempus lectus a nisl congue, ac venenatis diam viverra. Nam ac justo id nulla iaculis lobortis in eu ligula. Vivamus et ligula id sapien efficitur aliquet. Curabitur est justo, tempus vitae mollis quis, tincidunt vitae felis. Vestibulum molestie laoreet justo, nec mollis purus vulputate at."; + } diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 7e9d6189948b..80a36128430b 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -7,15 +7,15 @@ | Amazon Seller Partner | [![source-amazon-seller-partner](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-amazon-seller-partner%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-amazon-seller-partner) | | Amplitude | [![source-amplitude](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-amplitude%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-amplitude) | | Apify Dataset | [![source-amplitude](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-apify-dataset%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-apify-dataset) | -| AppsFlyer | [![source-braintree-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-appsflyer-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-appsflyer-singer) | +| AppsFlyer | [![source-appsflyer-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-appsflyer-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-appsflyer-singer) | | App Store | [![source-appstore-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-appstore-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-appstore-singer) | | Asana | [![source-asana](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-asana%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-asana) | | AWS CloudTrail | [![source-aws-cloudtrail](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-aws-cloudtrail%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-aws-cloudtrail) | -| BambooHR | [![source-bamboo-hr](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bamboo-hr%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-braintree) | +| BambooHR | [![source-bamboo-hr](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bamboo-hr%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bamboo-hr) | | Braintree | [![source-braintree](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-braintree%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-braintree) | -| BigCommerce | [![source-bigcommerce](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bigcommerce%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-braintree) | +| BigCommerce | [![source-bigcommerce](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bigcommerce%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bigcommerce) | | BigQuery | [![source-bigquery](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bigquery%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bigquery/) | -| Bing Ads [![source-bing-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-bing-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bing-ads) | +| Bing Ads | [![source-bing-ads](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-bing-ads%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-bing-ads) | | Chargebee | [![source-chargebee](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-chargebee%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-chargebee/) | | Cart.com | [![source-cart](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-cart%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-cart/) | | Close.com | [![source-close-com](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-close-com%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-close-com/) | @@ -36,15 +36,16 @@ | Google Directory API | [![source-google-directory](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-google-directory%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-google-directory) | | Google Workspace Admin | [![source-google-workspace-admin-reports](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-google-workspace-admin-reports%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-google-workspace-admin-reports) | | Greenhouse | [![source-greenhouse](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-greenhouse%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-greenhouse) | -| Hubspot | [![source-hubspot-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-hubspot%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-hubspot) | +| Hubspot | [![source-hubspot](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-hubspot%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-hubspot) | | IBM Db2 | [![source-db2](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-db2%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-db2) | | Instagram | [![source-instagram](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-instagram%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-instagram) | | Intercom | [![source-intercom](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-intercom-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-intercom) | | Iterable | [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-iterable) | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-jira) | +| Lever Hiring | [![source-lever-hiring](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-lever-hiring%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-lever-hiring) | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-looker) | | Klaviyo | [![source-klaviyo](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-klaviyo%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-klaviyo) | -| Kustomer | [![source-kustomer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-kustomer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-kustomer-singer) | +| Kustomer | [![source-kustomer-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-kustomer-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-kustomer-singer) | | Mailchimp | [![source-mailchimp](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mailchimp%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mailchimp) | | Marketo | [![source-marketo-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-marketo-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-marketo-singer) | | Microsoft SQL Server \(MSSQL\) | [![source-mssql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mssql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mssql) | @@ -54,7 +55,7 @@ | MySQL | [![source-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mysql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mysql) | | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | -| Pipedrive | [![source-pipedrive](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-pipedrive) | +| Pipedrive | [![source-pipedrive](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-pipedrive%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-pipedrive) | | Plaid | [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-plaid) | | Postgres | [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-postgres) | | PrestaShop | [![source-prestashop](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-prestashop%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-prestashop) | @@ -67,7 +68,7 @@ | Salesforce | [![source-salesforce](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-salesforce%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-salesforce) | | Sendgrid | [![source-sendgrid](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-sendgrid%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-sendgrid) | | Shopify | [![source-shopify](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-shopify%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-shopify) | -| Slack | [![source-slack-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-slack-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-slack-singer) | +| Slack | [![source-slack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-slack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-slack) | | Smartsheets | [![source-smartsheets](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-smartsheets%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-smartsheets) | | Snapchat Marketing | [![source-snapchat-marketing](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-snapchat-marketing%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-snapchat-marketing) | | Snowflake | [![source-snowflake](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-snowflake%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-snowflake) | @@ -82,7 +83,7 @@ | Zendesk Support | [![source-zendesk-support](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zendesk-support%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zendesk-support) | | Zendesk Talk | [![source-zendesk-talk](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zendesk-talk%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zendesk-talk) | | Zoom | [![source-zoom-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zoom-singer%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zoom-singer) | -| Zuora | [![source-zoom-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zuora%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zuora) | +| Zuora | [![source-zuora](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zuora%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zuora) | # Destinations @@ -91,7 +92,7 @@ | Azure Blob Storage | [![destination-azure-blob-storage](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-azure-blob-storage%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-azure-blob-storage) | | BigQuery | [![destination-bigquery](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-bigquery%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-bigquery) | | Databricks | (Temporarily Not Available) | -| Google Cloud Storage (GCS) | [![destination-gcs](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-s3%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-gcs) | +| Google Cloud Storage (GCS) | [![destination-gcs](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-gcs%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-gcs) | | Google PubSub | [![destination-pubsub](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-pubsub%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-pubsub) | | Kafka | [![destination-kafka](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-kafka%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-kafka) | | Keen (Chargify) | [![destination-keen](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fdestination-keen%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/destination-keen) | diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle index 24fe77e61cb7..56ad5d532b75 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle index a893b01823be..11b47deb667d 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-bigquery/README.md b/airbyte-integrations/connectors/destination-bigquery/README.md index d2fff147ed30..a1c5877eafcb 100644 --- a/airbyte-integrations/connectors/destination-bigquery/README.md +++ b/airbyte-integrations/connectors/destination-bigquery/README.md @@ -1,3 +1,67 @@ +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-bigquery:build +``` + +#### Create credentials +**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. +Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. + +**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci) to set up the credentials. + +### Locally running the connector docker image + +#### Build +Build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-bigquery:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-bigquery:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-bigquery:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-bigquery:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-bigquery:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +We use `JUnit` for Java tests. + +### Unit and Integration Tests +Place unit tests under `src/test/io/airbyte/integrations/destinations/bigquery`. + +#### Acceptance Tests +Airbyte has a standard test suite that all destination connectors must pass. Implement the `TODO`s in +`src/test-integration/java/io/airbyte/integrations/destinations/BigQueryDestinationAcceptanceTest.java`. + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-bigquery:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-bigquery:integrationTest +``` + +## Dependency Management + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + ## Uploading options There are 2 available options to upload data to bigquery `Standard` and `GCS Staging`. - `Standard` is option to upload data directly from your source to BigQuery storage. This way is faster and requires less resources than GCS one. @@ -18,23 +82,9 @@ In order to test the BigQuery destination, you need a service account key file. ## Community Contributor -As a community contributor, you will need access to a GCP project and BigQuery to run tests. - -1. Go to the `Service Accounts` page on the GCP console -1. Click on `+ Create Service Account" button -1. Fill out a descriptive name/id/description -1. Click the edit icon next to the service account you created on the `IAM` page -1. Add the `BigQuery Data Editor`, `BigQuery User` and `GCS User` roles. For more details check https://cloud.google.com/storage/docs/access-control/iam-roles -1. Go back to the `Service Accounts` page and use the actions modal to `Create Key` -1. Download this key as a JSON file -1. Create an GCS bucket for testing. -1. Generate a [HMAC key](https://cloud.google.com/storage/docs/authentication/hmackeys) for the bucket with reading and writing permissions. Please note that currently only the HMAC key credential is supported. More credential types will be added in the future. -1. Paste the bucket and key information into the config files under [`./sample_secret`](./sample_secret). -1. Rename the directory from `sample_secret` to `secrets`. -1. Feel free to modify the config files with different settings in the acceptance test file as long as they follow the schema defined in [spec.json](src/main/resources/spec.json). -1. Move and rename this file to `secrets/credentials.json` +Follow the setup guide in the [docs](https://docs.airbyte.io/integrations/destinations/bigquery) to obtain credentials. ## Airbyte Employee -1. Access the `BigQuery Integration Test User` secret on Rippling under the `Engineering` folder +1. Access the `BigQuery Integration Test User` secret on Lastpass under the `Engineering` folder 1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 9889647d59bc..86bf75b9cc55 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-csv/build.gradle b/airbyte-integrations/connectors/destination-csv/build.gradle index b515f922513d..e7ea3ece3987 100644 --- a/airbyte-integrations/connectors/destination-csv/build.gradle +++ b/airbyte-integrations/connectors/destination-csv/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.csv.CsvDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java index de0569675476..073461336788 100644 --- a/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java @@ -80,8 +80,10 @@ protected List retrieveRecords(TestDestinationEnv testEnv, JsonNode streamSchema) throws Exception { final List allOutputs = Files.list(testEnv.getLocalRoot().resolve(RELATIVE_PATH)).collect(Collectors.toList()); + final Optional streamOutput = - allOutputs.stream().filter(path -> path.getFileName().toString().contains(new StandardNameTransformer().getRawTableName(streamName))) + allOutputs.stream() + .filter(path -> path.getFileName().toString().endsWith(new StandardNameTransformer().getRawTableName(streamName) + ".csv")) .findFirst(); assertTrue(streamOutput.isPresent(), "could not find output file for stream: " + streamName); diff --git a/airbyte-integrations/connectors/destination-dynamodb/build.gradle b/airbyte-integrations/connectors/destination-dynamodb/build.gradle index b33317f137c7..e464bb647578 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/destination-dynamodb/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.dynamodb.DynamodbDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile index 158ac162a09f..f4141aa02fb7 100644 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ b/airbyte-integrations/connectors/destination-gcs/Dockerfile @@ -7,5 +7,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index 58b24c19a742..bf27f8686edc 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.gcs.GcsDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java index 60b7d30b8a0d..2c6e3322b1c6 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java @@ -73,7 +73,8 @@ public GcsAvroWriter(GcsDestinationConfig config, objectKey); this.avroRecordFactory = new AvroRecordFactory(schema, nameUpdater); - this.uploadManager = S3StreamTransferManagerHelper.getDefault(config.getBucketName(), objectKey, s3Client); + this.uploadManager = S3StreamTransferManagerHelper.getDefault( + config.getBucketName(), objectKey, s3Client, config.getFormatConfig().getPartSize()); // We only need one output stream as we only have one input stream. This is reasonably performant. this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java index 92570ba8d513..9ac0d1cc4ff3 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java @@ -74,7 +74,8 @@ public GcsCsvWriter(GcsDestinationConfig config, LOGGER.info("Full GCS path for stream '{}': {}/{}", stream.getName(), config.getBucketName(), objectKey); - this.uploadManager = S3StreamTransferManagerHelper.getDefault(config.getBucketName(), objectKey, s3Client); + this.uploadManager = S3StreamTransferManagerHelper.getDefault( + config.getBucketName(), objectKey, s3Client, config.getFormatConfig().getPartSize()); // We only need one output stream as we only have one input stream. This is reasonably performant. this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); this.csvPrinter = new CSVPrinter(new PrintWriter(outputStream, true, StandardCharsets.UTF_8), diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java index 690493790d3c..ed36b6b280d9 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java @@ -67,7 +67,9 @@ public GcsJsonlWriter(GcsDestinationConfig config, LOGGER.info("Full GCS path for stream '{}': {}/{}", stream.getName(), config.getBucketName(), objectKey); - this.uploadManager = S3StreamTransferManagerHelper.getDefault(config.getBucketName(), objectKey, s3Client); + this.uploadManager = S3StreamTransferManagerHelper.getDefault( + config.getBucketName(), objectKey, s3Client, config.getFormatConfig().getPartSize()); + // We only need one output stream as we only have one input stream. This is reasonably performant. this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); this.printWriter = new PrintWriter(outputStream, true, StandardCharsets.UTF_8); diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-gcs/src/main/resources/spec.json index 2e68885449d8..46c59eae5249 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-gcs/src/main/resources/spec.json @@ -229,6 +229,13 @@ } } ] + }, + "part_size_mb": { + "title": "Block Size (MB) for GCS multipart upload", + "description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.", + "type": "integer", + "default": 5, + "examples": [5] } } }, @@ -247,6 +254,13 @@ "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", "default": "No flattening", "enum": ["No flattening", "Root level flattening"] + }, + "part_size_mb": { + "title": "Block Size (MB) for GCS multipart upload", + "description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.", + "type": "integer", + "default": 5, + "examples": [5] } } }, @@ -258,6 +272,13 @@ "type": "string", "enum": ["JSONL"], "default": "JSONL" + }, + "part_size_mb": { + "title": "Block Size (MB) for GCS multipart upload", + "description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.", + "type": "integer", + "default": 5, + "examples": [5] } } }, diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java index d4341aee30df..99145665d05b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java @@ -24,6 +24,8 @@ package io.airbyte.integrations.destination.gcs; +import static io.airbyte.integrations.destination.s3.S3DestinationConstants.NAME_TRANSFORMER; + import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -107,6 +109,7 @@ protected List getAllSyncedObjects(String streamName, String na .listObjects(config.getBucketName(), outputPrefix) .getObjectSummaries() .stream() + .filter(o -> o.getKey().contains(NAME_TRANSFORMER.convertStreamName(streamName) + "/")) .sorted(Comparator.comparingLong(o -> o.getLastModified().getTime())) .collect(Collectors.toList()); LOGGER.info( diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java new file mode 100644 index 000000000000..fabb672a148e --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java @@ -0,0 +1,165 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.gcs.avro; + +import static com.amazonaws.services.s3.internal.Constants.MB; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import alex.mojaki.s3upload.StreamTransferManager; +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; +import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; +import io.airbyte.integrations.destination.s3.util.S3StreamTransferManagerHelper; +import java.util.List; +import org.apache.avro.file.CodecFactory; +import org.apache.avro.file.DataFileConstants; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class GcsAvroFormatConfigTest { + + @Test + public void testParseCodecConfigNull() { + List nullConfigs = Lists.newArrayList("{}", "{ \"codec\": \"no compression\" }"); + for (String nullConfig : nullConfigs) { + assertEquals( + DataFileConstants.NULL_CODEC, + S3AvroFormatConfig.parseCodecConfig(Jsons.deserialize(nullConfig)).toString()); + } + } + + @Test + public void testParseCodecConfigDeflate() { + // default compression level 0 + CodecFactory codecFactory1 = S3AvroFormatConfig.parseCodecConfig( + Jsons.deserialize("{ \"codec\": \"deflate\" }")); + assertEquals("deflate-0", codecFactory1.toString()); + + // compression level 5 + CodecFactory codecFactory2 = S3AvroFormatConfig.parseCodecConfig( + Jsons.deserialize("{ \"codec\": \"deflate\", \"compression_level\": 5 }")); + assertEquals("deflate-5", codecFactory2.toString()); + } + + @Test + public void testParseCodecConfigBzip2() { + JsonNode bzip2Config = Jsons.deserialize("{ \"codec\": \"bzip2\" }"); + CodecFactory codecFactory = S3AvroFormatConfig.parseCodecConfig(bzip2Config); + assertEquals(DataFileConstants.BZIP2_CODEC, codecFactory.toString()); + } + + @Test + public void testParseCodecConfigXz() { + // default compression level 6 + CodecFactory codecFactory1 = S3AvroFormatConfig.parseCodecConfig( + Jsons.deserialize("{ \"codec\": \"xz\" }")); + assertEquals("xz-6", codecFactory1.toString()); + + // compression level 7 + CodecFactory codecFactory2 = S3AvroFormatConfig.parseCodecConfig( + Jsons.deserialize("{ \"codec\": \"xz\", \"compression_level\": 7 }")); + assertEquals("xz-7", codecFactory2.toString()); + } + + @Test + public void testParseCodecConfigZstandard() { + // default compression level 3 + CodecFactory codecFactory1 = S3AvroFormatConfig.parseCodecConfig( + Jsons.deserialize("{ \"codec\": \"zstandard\" }")); + // There is no way to verify the checksum; all relevant methods are private or protected... + assertEquals("zstandard[3]", codecFactory1.toString()); + + // compression level 20 + CodecFactory codecFactory2 = S3AvroFormatConfig.parseCodecConfig( + Jsons.deserialize( + "{ \"codec\": \"zstandard\", \"compression_level\": 20, \"include_checksum\": true }")); + // There is no way to verify the checksum; all relevant methods are private or protected... + assertEquals("zstandard[20]", codecFactory2.toString()); + } + + @Test + public void testParseCodecConfigSnappy() { + JsonNode snappyConfig = Jsons.deserialize("{ \"codec\": \"snappy\" }"); + CodecFactory codecFactory = S3AvroFormatConfig.parseCodecConfig(snappyConfig); + assertEquals(DataFileConstants.SNAPPY_CODEC, codecFactory.toString()); + } + + @Test + public void testParseCodecConfigInvalid() { + Assertions.assertThrows(IllegalArgumentException.class, () -> { + JsonNode invalidConfig = Jsons.deserialize("{ \"codec\": \"bi-directional-bfs\" }"); + S3AvroFormatConfig.parseCodecConfig(invalidConfig); + }); + } + + @Test + public void testHandlePartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"AVRO\",\n" + + " \"part_size_mb\": 6\n" + + "}")); + + GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig + .getGcsDestinationConfig(config); + ConfigTestUtils.assertBaseConfig(gcsDestinationConfig); + + S3FormatConfig formatConfig = gcsDestinationConfig.getFormatConfig(); + assertEquals("AVRO", formatConfig.getFormat().name()); + assertEquals(6, formatConfig.getPartSize()); + // Assert that is set properly in config + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + gcsDestinationConfig.getBucketName(), "objectKey", null, + gcsDestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 6, partSizeBytes); + } + + @Test + public void testHandleAbsenceOfPartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"AVRO\"\n" + + "}")); + + GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig + .getGcsDestinationConfig(config); + ConfigTestUtils.assertBaseConfig(gcsDestinationConfig); + + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + gcsDestinationConfig.getBucketName(), "objectKey", null, + gcsDestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 5, partSizeBytes); // 5MB is a default value if nothing provided explicitly + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java new file mode 100644 index 000000000000..232815b931be --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java @@ -0,0 +1,103 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.gcs.csv; + +import static com.amazonaws.services.s3.internal.Constants.MB; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import alex.mojaki.s3upload.StreamTransferManager; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; +import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig.Flattening; +import io.airbyte.integrations.destination.s3.util.S3StreamTransferManagerHelper; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("GcsCsvFormatConfig") +public class GcsCsvFormatConfigTest { + + @Test + @DisplayName("Flattening enums can be created from value string") + public void testFlatteningCreationFromString() { + assertEquals(Flattening.NO, Flattening.fromValue("no flattening")); + assertEquals(Flattening.ROOT_LEVEL, Flattening.fromValue("root level flattening")); + try { + Flattening.fromValue("invalid flattening value"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + + @Test + public void testHandlePartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"CSV\",\n" + + " \"flattening\": \"Root level flattening\",\n" + + " \"part_size_mb\": 6\n" + + "}")); + + GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig + .getGcsDestinationConfig(config); + ConfigTestUtils.assertBaseConfig(gcsDestinationConfig); + + S3FormatConfig formatConfig = gcsDestinationConfig.getFormatConfig(); + assertEquals("CSV", formatConfig.getFormat().name()); + assertEquals(6, formatConfig.getPartSize()); + // Assert that is set properly in config + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + gcsDestinationConfig.getBucketName(), "objectKey", null, + gcsDestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 6, partSizeBytes); + } + + @Test + public void testHandleAbsenceOfPartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"CSV\",\n" + + " \"flattening\": \"Root level flattening\"\n" + + "}")); + + GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig + .getGcsDestinationConfig(config); + ConfigTestUtils.assertBaseConfig(gcsDestinationConfig); + + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + gcsDestinationConfig.getBucketName(), "objectKey", null, + gcsDestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 5, partSizeBytes); // 5MB is a default value if nothing provided explicitly + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java new file mode 100644 index 000000000000..130d23bf4609 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.gcs.jsonl; + +import static com.amazonaws.services.s3.internal.Constants.MB; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import alex.mojaki.s3upload.StreamTransferManager; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; +import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; +import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.integrations.destination.s3.util.S3StreamTransferManagerHelper; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("GcsJsonlFormatConfig") +public class GcsJsonlFormatConfigTest { + + @Test + public void testHandlePartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"JSONL\",\n" + + " \"part_size_mb\": 6\n" + + "}")); + + GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig + .getGcsDestinationConfig(config); + ConfigTestUtils.assertBaseConfig(gcsDestinationConfig); + + S3FormatConfig formatConfig = gcsDestinationConfig.getFormatConfig(); + assertEquals("JSONL", formatConfig.getFormat().name()); + assertEquals(6, formatConfig.getPartSize()); + + // Assert that is set properly in config + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + gcsDestinationConfig.getBucketName(), "objectKey", null, + gcsDestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 6, partSizeBytes); + } + + @Test + public void testHandleAbsenceOfPartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"JSONL\"\n" + + "}")); + + GcsDestinationConfig gcsDestinationConfig = GcsDestinationConfig + .getGcsDestinationConfig(config); + ConfigTestUtils.assertBaseConfig(gcsDestinationConfig); + + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + gcsDestinationConfig.getBucketName(), "objectKey", null, + gcsDestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 5, partSizeBytes); // 5MB is a default value if nothing provided explicitly + } + +} diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java new file mode 100644 index 000000000000..f4d142f77e66 --- /dev/null +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/util/ConfigTestUtils.java @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.gcs.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; + +public class ConfigTestUtils { + + public static JsonNode getBaseConfig(JsonNode formatConfig) { + return Jsons.deserialize("{\n" + + " \"gcs_bucket_name\": \"test-bucket-name\",\n" + + " \"gcs_bucket_path\": \"test_path\",\n" + + " \"gcs_bucket_region\": \"us-east-2\"," + + " \"credential\": {\n" + + " \"credential_type\": \"HMAC_KEY\",\n" + + " \"hmac_key_access_id\": \"some_hmac_key\",\n" + + " \"hmac_key_secret\": \"some_key_secret\"\n" + + " }," + + " \"format\": " + formatConfig + + "}"); + + } + + public static void assertBaseConfig(GcsDestinationConfig gcsDestinationConfig) { + assertEquals("test-bucket-name", gcsDestinationConfig.getBucketName()); + assertEquals("test_path", gcsDestinationConfig.getBucketPath()); + assertEquals("us-east-2", gcsDestinationConfig.getBucketRegion()); + } + +} diff --git a/airbyte-integrations/connectors/destination-kafka/build.gradle b/airbyte-integrations/connectors/destination-kafka/build.gradle index c020722fdeac..626b69e650a1 100644 --- a/airbyte-integrations/connectors/destination-kafka/build.gradle +++ b/airbyte-integrations/connectors/destination-kafka/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.kafka.KafkaDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-keen/build.gradle b/airbyte-integrations/connectors/destination-keen/build.gradle index 06ce337e309d..bf11a5205437 100644 --- a/airbyte-integrations/connectors/destination-keen/build.gradle +++ b/airbyte-integrations/connectors/destination-keen/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.keen.KeenDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-local-json/build.gradle b/airbyte-integrations/connectors/destination-local-json/build.gradle index bf40fb661631..c2b62e70e555 100644 --- a/airbyte-integrations/connectors/destination-local-json/build.gradle +++ b/airbyte-integrations/connectors/destination-local-json/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.local_json.LocalJsonDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java index fad8f6b77639..220b9316e04d 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java @@ -77,7 +77,7 @@ protected List retrieveRecords(TestDestinationEnv testEnv, throws Exception { final List allOutputs = Files.list(testEnv.getLocalRoot().resolve(RELATIVE_PATH)).collect(Collectors.toList()); final Optional streamOutput = allOutputs.stream() - .filter(path -> path.getFileName().toString().contains(new StandardNameTransformer().getRawTableName(streamName))) + .filter(path -> path.getFileName().toString().endsWith(new StandardNameTransformer().getRawTableName(streamName) + ".jsonl")) .findFirst(); assertTrue(streamOutput.isPresent(), "could not find output file for stream: " + streamName); diff --git a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile index 3985cb352b4e..cf8a9c7f2c9a 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/destination-meilisearch diff --git a/airbyte-integrations/connectors/destination-meilisearch/build.gradle b/airbyte-integrations/connectors/destination-meilisearch/build.gradle index 446eb5e4bfe9..72bfb0da6a8c 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/build.gradle +++ b/airbyte-integrations/connectors/destination-meilisearch/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.meilisearch.MeiliSearchDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java b/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java index d3582927e81a..2e16119acda5 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java +++ b/airbyte-integrations/connectors/destination-meilisearch/src/main/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestination.java @@ -45,6 +45,8 @@ import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.DestinationSyncMode; import java.time.Instant; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Arrays; import java.util.HashMap; import java.util.Map; @@ -83,8 +85,10 @@ public class MeiliSearchDestination extends BaseConnector implements Destination private static final Logger LOGGER = LoggerFactory.getLogger(MeiliSearchDestination.class); private static final int MAX_BATCH_SIZE = 10000; + private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSSSSSSS"); public static final String AB_PK_COLUMN = "_ab_pk"; + public static final String AB_EMITTED_AT_COLUMN = "_ab_emitted_at"; @Override public AirbyteConnectionStatus check(JsonNode config) { @@ -164,6 +168,7 @@ private static RecordWriter recordWriterFunction(final Map indexN .stream() .map(AirbyteRecordMessage::getData) .peek(o -> ((ObjectNode) o).put(AB_PK_COLUMN, Names.toAlphanumericAndUnderscore(UUID.randomUUID().toString()))) + .peek(o -> ((ObjectNode) o).put(AB_EMITTED_AT_COLUMN, LocalDateTime.now().format(FORMATTER))) .collect(Collectors.toList())); final String s = index.addDocuments(json); LOGGER.info("add docs response {}", s); diff --git a/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java index 643242f31fdc..501e3401da36 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-meilisearch/src/test-integration/java/io/airbyte/integrations/destination/meilisearch/MeiliSearchDestinationAcceptanceTest.java @@ -37,6 +37,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; import org.testcontainers.containers.GenericContainer; @@ -104,7 +105,12 @@ protected List retrieveRecords(TestDestinationEnv env, return MoreStreams.toStream(response.iterator()) // strip out the airbyte primary key because the test cases only expect the data, no the airbyte // metadata column. + // We also sort the data by "emitted_at" and then remove that column, because the test cases only + // expect data, + // not the airbyte metadata column. .peek(r -> ((ObjectNode) r).remove(MeiliSearchDestination.AB_PK_COLUMN)) + .sorted(Comparator.comparing(o -> o.get(MeiliSearchDestination.AB_EMITTED_AT_COLUMN).asText())) + .peek(r -> ((ObjectNode) r).remove(MeiliSearchDestination.AB_EMITTED_AT_COLUMN)) .collect(Collectors.toList()); } diff --git a/airbyte-integrations/connectors/destination-mongodb/build.gradle b/airbyte-integrations/connectors/destination-mongodb/build.gradle index 5c7740de8031..1a59ea8e51a2 100644 --- a/airbyte-integrations/connectors/destination-mongodb/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-mssql/build.gradle b/airbyte-integrations/connectors/destination-mssql/build.gradle index 1f338f84c8d4..34c8a691f0cf 100644 --- a/airbyte-integrations/connectors/destination-mssql/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.mssql.MSSQLDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-mysql/build.gradle b/airbyte-integrations/connectors/destination-mysql/build.gradle index ddc981044a0a..111935e4eb9a 100644 --- a/airbyte-integrations/connectors/destination-mysql/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.mysql.MySQLDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index d9a8bbc3bf94..611e1e26a211 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.oracle.OracleDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index 4b1d32aa0efd..93005ae96a84 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-pubsub/build.gradle b/airbyte-integrations/connectors/destination-pubsub/build.gradle index 8830c4b8b747..ce699beba596 100644 --- a/airbyte-integrations/connectors/destination-pubsub/build.gradle +++ b/airbyte-integrations/connectors/destination-pubsub/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.pubsub.PubsubDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 54052f1e25f7..bfb9867e23b8 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.redshift.RedshiftDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } repositories { diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile index 85327fbe2b80..c27f60d88243 100644 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -7,5 +7,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index bb0ce530c9c6..6900ba3e8112 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.s3.S3Destination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java index ae1115a027da..868ee9d232f1 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java @@ -30,6 +30,7 @@ public final class S3DestinationConstants { public static final String YYYY_MM_DD_FORMAT_STRING = "yyyy_MM_dd"; public static final ExtendedNameTransformer NAME_TRANSFORMER = new ExtendedNameTransformer(); + public static final String PART_SIZE_MB_ARG_NAME = "part_size_mb"; private S3DestinationConstants() {} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java index 51a216ec3e69..c198da8b17d0 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java @@ -30,6 +30,8 @@ public interface S3FormatConfig { S3Format getFormat(); + Long getPartSize(); + static String withDefault(JsonNode config, String property, String defaultValue) { JsonNode value = config.get(property); if (value == null || value.isNull()) { diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java index d969846100c6..f38f62144bd1 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java @@ -50,7 +50,7 @@ public static S3FormatConfig getS3FormatConfig(JsonNode config) { return new S3CsvFormatConfig(formatConfig); } case JSONL -> { - return new S3JsonlFormatConfig(); + return new S3JsonlFormatConfig(formatConfig); } case PARQUET -> { return new S3ParquetFormatConfig(formatConfig); diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java index 95a68020fa86..aeb57b78bcce 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java @@ -24,6 +24,8 @@ package io.airbyte.integrations.destination.s3.avro; +import static io.airbyte.integrations.destination.s3.S3DestinationConstants.PART_SIZE_MB_ARG_NAME; + import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.destination.s3.S3Format; import io.airbyte.integrations.destination.s3.S3FormatConfig; @@ -32,9 +34,11 @@ public class S3AvroFormatConfig implements S3FormatConfig { private final CodecFactory codecFactory; + private final Long partSize; public S3AvroFormatConfig(JsonNode formatConfig) { this.codecFactory = parseCodecConfig(formatConfig.get("compression_codec")); + this.partSize = formatConfig.get(PART_SIZE_MB_ARG_NAME) != null ? formatConfig.get(PART_SIZE_MB_ARG_NAME).asLong() : null; } public static CodecFactory parseCodecConfig(JsonNode compressionCodecConfig) { @@ -102,6 +106,10 @@ public CodecFactory getCodecFactory() { return codecFactory; } + public Long getPartSize() { + return partSize; + } + @Override public S3Format getFormat() { return S3Format.AVRO; diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java index 317f434caa30..b16380c8895a 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java @@ -70,7 +70,8 @@ public S3AvroWriter(S3DestinationConfig config, objectKey); this.avroRecordFactory = new AvroRecordFactory(schema, nameUpdater); - this.uploadManager = S3StreamTransferManagerHelper.getDefault(config.getBucketName(), objectKey, s3Client); + this.uploadManager = S3StreamTransferManagerHelper.getDefault( + config.getBucketName(), objectKey, s3Client, config.getFormatConfig().getPartSize()); // We only need one output stream as we only have one input stream. This is reasonably performant. this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java index 00c4d07b8c83..35ed863df4a0 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java @@ -24,6 +24,8 @@ package io.airbyte.integrations.destination.s3.csv; +import static io.airbyte.integrations.destination.s3.S3DestinationConstants.PART_SIZE_MB_ARG_NAME; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.destination.s3.S3Format; @@ -60,9 +62,11 @@ public String getValue() { } private final Flattening flattening; + private final Long partSize; public S3CsvFormatConfig(JsonNode formatConfig) { this.flattening = Flattening.fromValue(formatConfig.get("flattening").asText()); + this.partSize = formatConfig.get(PART_SIZE_MB_ARG_NAME) != null ? formatConfig.get(PART_SIZE_MB_ARG_NAME).asLong() : null; } @Override @@ -74,10 +78,15 @@ public Flattening getFlattening() { return flattening; } + public Long getPartSize() { + return partSize; + } + @Override public String toString() { return "S3CsvFormatConfig{" + "flattening=" + flattening + + ", partSize=" + partSize + '}'; } diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java index 5a477b92c054..7f067e5ce180 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java @@ -71,7 +71,8 @@ public S3CsvWriter(S3DestinationConfig config, LOGGER.info("Full S3 path for stream '{}': s3://{}/{}", stream.getName(), config.getBucketName(), objectKey); - this.uploadManager = S3StreamTransferManagerHelper.getDefault(config.getBucketName(), objectKey, s3Client); + this.uploadManager = S3StreamTransferManagerHelper.getDefault( + config.getBucketName(), objectKey, s3Client, config.getFormatConfig().getPartSize()); // We only need one output stream as we only have one input stream. This is reasonably performant. this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); this.csvPrinter = new CSVPrinter(new PrintWriter(outputStream, true, StandardCharsets.UTF_8), diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java index 15ec7b5684d3..a2b58125653d 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java @@ -24,14 +24,27 @@ package io.airbyte.integrations.destination.s3.jsonl; +import static io.airbyte.integrations.destination.s3.S3DestinationConstants.PART_SIZE_MB_ARG_NAME; + +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.destination.s3.S3Format; import io.airbyte.integrations.destination.s3.S3FormatConfig; public class S3JsonlFormatConfig implements S3FormatConfig { + private final Long partSize; + + public S3JsonlFormatConfig(JsonNode formatConfig) { + this.partSize = formatConfig.get(PART_SIZE_MB_ARG_NAME) != null ? formatConfig.get(PART_SIZE_MB_ARG_NAME).asLong() : null; + } + @Override public S3Format getFormat() { return S3Format.JSONL; } + public Long getPartSize() { + return partSize; + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java index fa3c4d4cbfbe..d55f82d0ffa7 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java @@ -70,7 +70,8 @@ public S3JsonlWriter(S3DestinationConfig config, LOGGER.info("Full S3 path for stream '{}': s3://{}/{}", stream.getName(), config.getBucketName(), objectKey); - this.uploadManager = S3StreamTransferManagerHelper.getDefault(config.getBucketName(), objectKey, s3Client); + this.uploadManager = S3StreamTransferManagerHelper.getDefault( + config.getBucketName(), objectKey, s3Client, config.getFormatConfig().getPartSize()); // We only need one output stream as we only have one input stream. This is reasonably performant. this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); this.printWriter = new PrintWriter(outputStream, true, StandardCharsets.UTF_8); diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java index 72cd49342afa..f73b5b99d1dc 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java @@ -59,6 +59,12 @@ public S3Format getFormat() { return S3Format.PARQUET; } + @Override + public Long getPartSize() { + // not applicable for Parquet format + return null; + } + public CompressionCodecName getCompressionCodec() { return compressionCodec; } diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3StreamTransferManagerHelper.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3StreamTransferManagerHelper.java index dc188536113c..d9361da6400d 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3StreamTransferManagerHelper.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3StreamTransferManagerHelper.java @@ -26,9 +26,13 @@ import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class S3StreamTransferManagerHelper { + protected static final Logger LOGGER = LoggerFactory.getLogger(S3StreamTransferManagerHelper.class); + // See this doc about how they affect memory usage: // https://alexmojaki.github.io/s3-stream-upload/javadoc/apidocs/alex/mojaki/s3upload/StreamTransferManager.html // Total memory = (numUploadThreads + queueCapacity) * partSize + numStreams * (partSize + 6MB) @@ -36,9 +40,47 @@ public class S3StreamTransferManagerHelper { public static final int DEFAULT_UPLOAD_THREADS = 2; public static final int DEFAULT_QUEUE_CAPACITY = 2; public static final int DEFAULT_PART_SIZE_MB = 5; + // MAX object size for AWS and GCS is 5TB (max allowed 10,000 parts*525mb) + // (https://aws.amazon.com/s3/faqs/, https://cloud.google.com/storage/quotas) + public static final int MAX_ALLOWED_PART_SIZE_MB = 525; public static final int DEFAULT_NUM_STREAMS = 1; - public static StreamTransferManager getDefault(String bucketName, String objectKey, AmazonS3 s3Client) { + public static StreamTransferManager getDefault(String bucketName, String objectKey, AmazonS3 s3Client, Long partSize) { + if (partSize == null) { + LOGGER.warn(String.format("Part size for StreamTransferManager is not set explicitly. Will use the default one = %sMB. " + + "Please note server allows up to 10,000 parts to be uploaded for a single object, i.e. 50GB for stream. " + + "Feel free to increase partSize arg, but make sure you have enough memory resources allocated", DEFAULT_PART_SIZE_MB)); + return getDefault(bucketName, objectKey, s3Client); + } + if (partSize < DEFAULT_PART_SIZE_MB) { + LOGGER.warn(String.format("By the server limitation part size can't be less than %sMB which is already set by default. " + + "Will use the default value", DEFAULT_PART_SIZE_MB)); + return getDefault(bucketName, objectKey, s3Client); + } + if (partSize > MAX_ALLOWED_PART_SIZE_MB) { + LOGGER.warn( + "Server allows up to 10,000 parts to be uploaded for a single object, and each part must be identified by a unique number from 1 to 10,000." + + " These part numbers are allocated evenly by the manager to each output stream. Therefore the maximum amount of" + + " data that can be written to a stream is 10000/numStreams * partSize. If you try to write more, an IndexOutOfBoundsException" + + " will be thrown. The total object size can be at most 5 TB, so there is no reason to set this higher" + + " than 525MB. If you're using more streams, you may want a higher value in case some streams get more data than others. " + + "So will use max allowed value =" + MAX_ALLOWED_PART_SIZE_MB); + return new StreamTransferManager(bucketName, objectKey, s3Client) + .numStreams(DEFAULT_NUM_STREAMS) + .queueCapacity(DEFAULT_QUEUE_CAPACITY) + .numUploadThreads(DEFAULT_UPLOAD_THREADS) + .partSize(MAX_ALLOWED_PART_SIZE_MB); + } + + LOGGER.info(String.format("PartSize arg is set to %s MB", partSize)); + return new StreamTransferManager(bucketName, objectKey, s3Client) + .numStreams(DEFAULT_NUM_STREAMS) + .queueCapacity(DEFAULT_QUEUE_CAPACITY) + .numUploadThreads(DEFAULT_UPLOAD_THREADS) + .partSize(partSize); + } + + private static StreamTransferManager getDefault(String bucketName, String objectKey, AmazonS3 s3Client) { // The stream transfer manager lets us greedily stream into S3. The native AWS SDK does not // have support for streaming multipart uploads. The alternative is first writing the entire // output to disk before loading into S3. This is not feasible with large input. diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json index f3ae3a1c9452..ceb0d8998cf2 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -200,6 +200,13 @@ } } ] + }, + "part_size_mb": { + "title": "Block Size (MB) for Amazon S3 multipart upload", + "description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.", + "type": "integer", + "default": 5, + "examples": [5] } } }, @@ -218,6 +225,13 @@ "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", "default": "No flattening", "enum": ["No flattening", "Root level flattening"] + }, + "part_size_mb": { + "title": "Block Size (MB) for Amazon S3 multipart upload", + "description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.", + "type": "integer", + "default": 5, + "examples": [5] } } }, @@ -229,6 +243,13 @@ "type": "string", "enum": ["JSONL"], "default": "JSONL" + }, + "part_size_mb": { + "title": "Block Size (MB) for Amazon S3 multipart upload", + "description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.", + "type": "integer", + "default": 5, + "examples": [5] } } }, diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java index b0fa3f564ec2..4762071ebc9b 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java @@ -24,15 +24,22 @@ package io.airbyte.integrations.destination.s3.avro; +import static com.amazonaws.services.s3.internal.Constants.MB; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; +import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.integrations.destination.s3.util.ConfigTestUtils; +import io.airbyte.integrations.destination.s3.util.S3StreamTransferManagerHelper; import java.util.List; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileConstants; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.Test; class S3AvroFormatConfigTest { @@ -90,7 +97,8 @@ public void testParseCodecConfigZstandard() { // compression level 20 CodecFactory codecFactory2 = S3AvroFormatConfig.parseCodecConfig( - Jsons.deserialize("{ \"codec\": \"zstandard\", \"compression_level\": 20, \"include_checksum\": true }")); + Jsons.deserialize( + "{ \"codec\": \"zstandard\", \"compression_level\": 20, \"include_checksum\": true }")); // There is no way to verify the checksum; all relevant methods are private or protected... assertEquals("zstandard[20]", codecFactory2.toString()); } @@ -113,4 +121,47 @@ public void testParseCodecConfigInvalid() { } } + @Test + public void testHandlePartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"AVRO\",\n" + + " \"part_size_mb\": 6\n" + + "}")); + + S3DestinationConfig s3DestinationConfig = S3DestinationConfig + .getS3DestinationConfig(config); + ConfigTestUtils.assertBaseConfig(s3DestinationConfig); + + S3FormatConfig formatConfig = s3DestinationConfig.getFormatConfig(); + assertEquals("AVRO", formatConfig.getFormat().name()); + assertEquals(6, formatConfig.getPartSize()); + // Assert that is set properly in config + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + s3DestinationConfig.getBucketName(), "objectKey", null, + s3DestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 6, partSizeBytes); + } + + @Test + public void testHandleAbsenceOfPartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"AVRO\"\n" + + "}")); + + S3DestinationConfig s3DestinationConfig = S3DestinationConfig + .getS3DestinationConfig(config); + ConfigTestUtils.assertBaseConfig(s3DestinationConfig); + + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + s3DestinationConfig.getBucketName(), "objectKey", null, + s3DestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 5, partSizeBytes); // 5MB is a default value if nothing provided explicitly + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java index 35dde8bb970c..c671652e877e 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java @@ -24,9 +24,18 @@ package io.airbyte.integrations.destination.s3.csv; +import static com.amazonaws.services.s3.internal.Constants.MB; import static org.junit.jupiter.api.Assertions.*; +import alex.mojaki.s3upload.StreamTransferManager; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.integrations.destination.s3.S3FormatConfig; import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig.Flattening; +import io.airbyte.integrations.destination.s3.util.ConfigTestUtils; +import io.airbyte.integrations.destination.s3.util.S3StreamTransferManagerHelper; +import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -45,4 +54,49 @@ public void testFlatteningCreationFromString() { } } + @Test + public void testHandlePartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"CSV\",\n" + + " \"flattening\": \"Root level flattening\",\n" + + " \"part_size_mb\": 6\n" + + "}")); + + S3DestinationConfig s3DestinationConfig = S3DestinationConfig + .getS3DestinationConfig(config); + ConfigTestUtils.assertBaseConfig(s3DestinationConfig); + + S3FormatConfig formatConfig = s3DestinationConfig.getFormatConfig(); + assertEquals("CSV", formatConfig.getFormat().name()); + assertEquals(6, formatConfig.getPartSize()); + // Assert that is set properly in config + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + s3DestinationConfig.getBucketName(), "objectKey", null, + s3DestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 6, partSizeBytes); + } + + @Test + public void testHandleAbsenceOfPartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"CSV\",\n" + + " \"flattening\": \"Root level flattening\"\n" + + "}")); + + S3DestinationConfig s3DestinationConfig = S3DestinationConfig + .getS3DestinationConfig(config); + ConfigTestUtils.assertBaseConfig(s3DestinationConfig); + + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + s3DestinationConfig.getBucketName(), "objectKey", null, + s3DestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 5, partSizeBytes); // 5MB is a default value if nothing provided explicitly + } + } diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java new file mode 100644 index 000000000000..523d1dbc7d1c --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.jsonl; + +import static com.amazonaws.services.s3.internal.Constants.MB; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import alex.mojaki.s3upload.StreamTransferManager; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.integrations.destination.s3.util.ConfigTestUtils; +import io.airbyte.integrations.destination.s3.util.S3StreamTransferManagerHelper; +import org.apache.commons.lang3.reflect.FieldUtils; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("S3JsonlFormatConfig") +public class S3JsonlFormatConfigTest { + + @Test + public void testHandlePartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"JSONL\",\n" + + " \"part_size_mb\": 6\n" + + "}")); + + S3DestinationConfig s3DestinationConfig = S3DestinationConfig + .getS3DestinationConfig(config); + ConfigTestUtils.assertBaseConfig(s3DestinationConfig); + + S3FormatConfig formatConfig = s3DestinationConfig.getFormatConfig(); + assertEquals("JSONL", formatConfig.getFormat().name()); + assertEquals(6, formatConfig.getPartSize()); + + // Assert that is set properly in config + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + s3DestinationConfig.getBucketName(), "objectKey", null, + s3DestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 6, partSizeBytes); + } + + @Test + public void testHandleAbsenceOfPartSizeConfig() throws IllegalAccessException { + + JsonNode config = ConfigTestUtils.getBaseConfig(Jsons.deserialize("{\n" + + " \"format_type\": \"JSONL\"\n" + + "}")); + + S3DestinationConfig s3DestinationConfig = S3DestinationConfig + .getS3DestinationConfig(config); + ConfigTestUtils.assertBaseConfig(s3DestinationConfig); + + StreamTransferManager streamTransferManager = S3StreamTransferManagerHelper.getDefault( + s3DestinationConfig.getBucketName(), "objectKey", null, + s3DestinationConfig.getFormatConfig().getPartSize()); + + Integer partSizeBytes = (Integer) FieldUtils.readField(streamTransferManager, "partSize", true); + assertEquals(MB * 5, partSizeBytes); // 5MB is a default value if nothing provided explicitly + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/util/ConfigTestUtils.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/util/ConfigTestUtils.java new file mode 100644 index 000000000000..a577f8d9b224 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/util/ConfigTestUtils.java @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.s3.S3DestinationConfig; + +public class ConfigTestUtils { + + public static JsonNode getBaseConfig(JsonNode formatConfig) { + return Jsons.deserialize("{\n" + + " \"s3_endpoint\": \"some_test-endpoint\",\n" + + " \"s3_bucket_name\": \"test-bucket-name\",\n" + + " \"s3_bucket_path\": \"test_path\",\n" + + " \"s3_bucket_region\": \"us-east-2\",\n" + + " \"access_key_id\": \"some-test-key-id\",\n" + + " \"secret_access_key\": \"some-test-access-key\",\n" + + " \"format\": " + formatConfig + + "}"); + } + + public static void assertBaseConfig(S3DestinationConfig s3DestinationConfig) { + assertEquals("some_test-endpoint", s3DestinationConfig.getEndpoint()); + assertEquals("test-bucket-name", s3DestinationConfig.getBucketName()); + assertEquals("test_path", s3DestinationConfig.getBucketPath()); + assertEquals("us-east-2", s3DestinationConfig.getBucketRegion()); + assertEquals("some-test-key-id", s3DestinationConfig.getAccessKeyId()); + assertEquals("some-test-access-key", s3DestinationConfig.getSecretAccessKey()); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index a52d143a1690..afb746dcbf4d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.destination.snowflake.SnowflakeDestination' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index 938385f17883..ad37646fb46a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py index ecf3d97e10d3..fa39703b6f64 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/profiles.py @@ -40,7 +40,7 @@ class Profiles(AmazonAdsStream): model = Profile def path(self, **kvargs) -> str: - return "v2/profiles" + return "v2/profiles?profileTypeFilter=seller,vendor" def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: for record in super().parse_response(response, **kwargs): diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile index f0efc4de3fd8..7b4fb4d33aeb 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index 81ad02e7970d..9e00b6929575 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -3,22 +3,38 @@ tests: spec: - spec_path: "source_amazon_seller_partner/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + timeout_seconds: 60 + - config_path: "integration_tests/invalid_config.json" + status: "failed" + timeout_seconds: 60 discovery: - - config_path: "secrets/config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_no_orders.json" - empty_streams: [] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_no_orders.json" - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL: ["createdTime"] - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" +# TODO: uncomment when at least one record exist +# basic_read: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# empty_streams: +# [ +# "Orders", +# "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", +# "GET_MERCHANT_LISTINGS_ALL_DATA", +# "GET_FBA_INVENTORY_AGED_DATA", +# "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", +# "GET_FLAT_FILE_OPEN_LISTINGS_DATA", +# "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA", +# "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA", +# "GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT", +# "VendorDirectFulfillmentShipping", +# ] +# TODO: uncomment when Orders (or any other incremental) stream is filled with data +# incremental: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/future_state.json" +# cursor_paths: +# Orders: ["LastUpdateDate"] +# full_refresh: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/abnormal_state.json deleted file mode 100644 index d97bb92f073a..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/abnormal_state.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "Orders": { - "LastUpdateDate": "2121-07-01T00:00:00Z" - }, - "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL": { - "createdTime": "2121-07-01T00:00:00Z" - }, - "GET_MERCHANT_LISTINGS_ALL_DATA": { - "createdTime": "2121-07-01T00:00:00Z" - }, - "GET_FBA_INVENTORY_AGED_DATA": { - "createdTime": "2121-07-01T00:00:00Z" - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog.json index cd4fc5c65588..0d2036119ae3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog.json @@ -16,37 +16,82 @@ "stream": { "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdTime"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["createdTime"] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { "name": "GET_MERCHANT_LISTINGS_ALL_DATA", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdTime"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["createdTime"] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { "name": "GET_FBA_INVENTORY_AGED_DATA", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdTime"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["createdTime"] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "VendorDirectFulfillmentShipping", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json new file mode 100644 index 000000000000..3542eab16f4e --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_FBA_INVENTORY_AGED_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_orders.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_orders.json deleted file mode 100644 index 4a81e10d100e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_orders.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdTime"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["createdTime"] - }, - { - "stream": { - "name": "GET_MERCHANT_LISTINGS_ALL_DATA", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdTime"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["createdTime"] - }, - { - "stream": { - "name": "GET_FBA_INVENTORY_AGED_DATA", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["createdTime"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["createdTime"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json new file mode 100644 index 000000000000..2676de94b9bf --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json @@ -0,0 +1,5 @@ +{ + "Orders": { + "LastUpdateDate": "2121-07-01T00:00:00Z" + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json index 17a8214ef89a..835aca988e7e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json @@ -10,5 +10,20 @@ }, "GET_FBA_INVENTORY_AGED_DATA": { "createdTime": "2021-07-01T00:00:00Z" + }, + "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL": { + "createdTime": "2021-07-01T00:00:00Z" + }, + "GET_FLAT_FILE_OPEN_LISTINGS_DATA": { + "createdTime": "2021-07-01T00:00:00Z" + }, + "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA": { + "createdTime": "2021-07-01T00:00:00Z" + }, + "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA": { + "createdTime": "2021-07-01T00:00:00Z" + }, + "GET_VENDOR_INVENTORY_HEALTH_REPORT": { + "createdTime": "2021-07-01T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index 3bbbd468a787..82f5a77ac713 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -25,7 +25,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1", "pycryptodome~=3.10"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index 2c1b361c42a4..01b28176b917 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -26,12 +26,12 @@ import hmac import urllib.parse from typing import Any, Mapping +from urllib.parse import urlparse import pendulum import requests from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator from requests.auth import AuthBase -from requests.compat import urlparse class AWSAuthenticator(Oauth2Authenticator): diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json new file mode 100644 index 000000000000..214ac2b7c020 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json @@ -0,0 +1,152 @@ +{ + "title": "Amazon Fulfilled Data General", + "description": "Amazon Fulfilled Data General Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "amazon-order-id": { + "type": ["null", "string"] + }, + "merchant-order-id": { + "type": ["null", "string"] + }, + "shipment-id": { + "type": ["null", "string"] + }, + "shipment-item-id": { + "type": ["null", "string"] + }, + "amazon-order-item-id": { + "type": ["null", "string"] + }, + "merchant-order-item-id": { + "type": ["null", "string"] + }, + "purchase-date": { + "type": ["null", "string"] + }, + "payments-date": { + "type": ["null", "string"] + }, + "shipment-date": { + "type": ["null", "string"] + }, + "reporting-date": { + "type": ["null", "string"] + }, + "buyer-email": { + "type": ["null", "string"] + }, + "buyer-name": { + "type": ["null", "string"] + }, + "buyer-phone-number": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "product-name": { + "type": ["null", "string"] + }, + "quantity-shipped": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "item-price": { + "type": ["null", "string"] + }, + "item-tax": { + "type": ["null", "string"] + }, + "shipping-price": { + "type": ["null", "string"] + }, + "shipping-tax": { + "type": ["null", "string"] + }, + "gift-wrap-price": { + "type": ["null", "string"] + }, + "gift-wrap-tax": { + "type": ["null", "string"] + }, + "ship-service-level": { + "type": ["null", "string"] + }, + "recipient-name": { + "type": ["null", "string"] + }, + "ship-address-1": { + "type": ["null", "string"] + }, + "ship-address-2": { + "type": ["null", "string"] + }, + "ship-address-3": { + "type": ["null", "string"] + }, + "ship-city": { + "type": ["null", "string"] + }, + "ship-state": { + "type": ["null", "string"] + }, + "ship-postal-code": { + "type": ["null", "string"] + }, + "ship-country": { + "type": ["null", "string"] + }, + "ship-phone-number": { + "type": ["null", "string"] + }, + "bill-address-1": { + "type": ["null", "string"] + }, + "bill-address-2": { + "type": ["null", "string"] + }, + "bill-address-3": { + "type": ["null", "string"] + }, + "bill-city": { + "type": ["null", "string"] + }, + "bill-state": { + "type": ["null", "string"] + }, + "bill-postal-code": { + "type": ["null", "string"] + }, + "bill-country": { + "type": ["null", "string"] + }, + "item-promotion-discount": { + "type": ["null", "string"] + }, + "ship-promotion-discount": { + "type": ["null", "string"] + }, + "carrier": { + "type": ["null", "string"] + }, + "tracking-number": { + "type": ["null", "string"] + }, + "estimated-arrival-date": { + "type": ["null", "string"] + }, + "fulfillment-center-id": { + "type": ["null", "string"] + }, + "fulfillment-channel": { + "type": ["null", "string"] + }, + "sales-channel": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json new file mode 100644 index 000000000000..7da807eac263 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json @@ -0,0 +1,53 @@ +{ + "title": "FBA Fulfillment Removal Order Detail Data", + "description": "FBA Fulfillment Removal Order Detail Data Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "request-date": { + "type": ["null", "string"] + }, + "order-id": { + "type": ["null", "string"] + }, + "order-type": { + "type": ["null", "string"] + }, + "order-status": { + "type": ["null", "string"] + }, + "last-updated-date": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "fnsku": { + "type": ["null", "string"] + }, + "disposition": { + "type": ["null", "string"] + }, + "requested-quantity": { + "type": ["null", "string"] + }, + "cancelled-quantity": { + "type": ["null", "string"] + }, + "disposed-quantity": { + "type": ["null", "string"] + }, + "shipped-quantity": { + "type": ["null", "string"] + }, + "in-process-quantity": { + "type": ["null", "string"] + }, + "removal-fee": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json new file mode 100644 index 000000000000..f31a80bd0d1e --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json @@ -0,0 +1,35 @@ +{ + "title": "FBA Fulfillment Removal Shipment Detail Data", + "description": "FBA Fulfillment Removal Shipment Detail Data Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "removal-date": { + "type": ["null", "string"] + }, + "order-id": { + "type": ["null", "string"] + }, + "shipment-date": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "fnsku": { + "type": ["null", "string"] + }, + "disposition": { + "type": ["null", "string"] + }, + "quantity shipped": { + "type": ["null", "string"] + }, + "carrier": { + "type": ["null", "string"] + }, + "tracking-number": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json index 08d69408eaff..430869075c75 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json @@ -1,46 +1,68 @@ { "title": "FBA Inventory Aged Data Reports", "description": "FBA Inventory Aged Data Reports", - "type": ["null", "object"], + "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "reportType": { + "sku": { "type": ["null", "string"] }, - "processingEndTime": { - "type": ["null", "string"], - "format": "date-time" + "fnsku": { + "type": ["null", "string"] + }, + "asin": { + "type": ["null", "string"] + }, + "product-name": { + "type": ["null", "string"] + }, + "condition": { + "type": ["null", "string"] + }, + "your-price": { + "type": ["null", "string"] + }, + "mfn-listing-exists": { + "type": ["null", "string"] + }, + "mfn-fulfillable-quantity": { + "type": ["null", "string"] }, - "processingStatus": { + "afn-listing-exists": { "type": ["null", "string"] }, - "marketplaceIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "afn-warehouse-quantity": { + "type": ["null", "string"] }, - "reportDocumentId": { + "afn-fulfillable-quantity": { "type": ["null", "string"] }, - "reportId": { + "afn-unsellable-quantity": { "type": ["null", "string"] }, - "dataEndTime": { - "type": ["null", "string"], - "format": "date-time" + "afn-reserved-quantity": { + "type": ["null", "string"] }, - "createdTime": { - "type": ["null", "string"], - "format": "date-time" + "afn-total-quantity": { + "type": ["null", "string"] }, - "processingStartTime": { - "type": ["null", "string"], - "format": "date-time" + "per-unit-volume": { + "type": ["null", "string"] }, - "dataStartTime": { - "type": ["null", "string"], - "format": "date-time" + "afn-inbound-working-quantity": { + "type": ["null", "string"] + }, + "afn-inbound-shipped-quantity": { + "type": ["null", "string"] + }, + "afn-inbound-receiving-quantity": { + "type": ["null", "string"] + }, + "afn-future-supply-buyable": { + "type": ["null", "string"] + }, + "afn-reserved-future-supply": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json index e6b494278fbd..374434b39d80 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json @@ -1,46 +1,188 @@ { "title": "Flat File All Orders Data Reports", "description": "Flat File All Orders Data by Order Date General Reports", - "type": ["null", "object"], + "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "reportType": { + "order-id": { "type": ["null", "string"] }, - "processingEndTime": { - "type": ["null", "string"], - "format": "date-time" + "order-item-id": { + "type": ["null", "string"] + }, + "purchase-date": { + "type": ["null", "string"] + }, + "payments-date": { + "type": ["null", "string"] + }, + "buyer-email": { + "type": ["null", "string"] + }, + "buyer-name": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "product-name": { + "type": ["null", "string"] + }, + "quantity-purchased": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "item-price": { + "type": ["null", "string"] + }, + "shipping-price": { + "type": ["null", "string"] + }, + "item-tax": { + "type": ["null", "string"] + }, + "ship-service-level": { + "type": ["null", "string"] + }, + "recipient-name": { + "type": ["null", "string"] + }, + "ship-address-1": { + "type": ["null", "string"] + }, + "ship-address-2": { + "type": ["null", "string"] + }, + "ship-address-3": { + "type": ["null", "string"] + }, + "ship-city": { + "type": ["null", "string"] + }, + "ship-state": { + "type": ["null", "string"] + }, + "ship-postal-code": { + "type": ["null", "string"] + }, + "ship-country": { + "type": ["null", "string"] + }, + "gift-wrap-type": { + "type": ["null", "string"] + }, + "gift-message-text": { + "type": ["null", "string"] + }, + "gift-wrap-price": { + "type": ["null", "string"] + }, + "gift-wrap-tax": { + "type": ["null", "string"] + }, + "item-promotion-discount": { + "type": ["null", "string"] + }, + "item-promotion-id": { + "type": ["null", "string"] }, - "processingStatus": { + "shipping-promotion-discount": { "type": ["null", "string"] }, - "marketplaceIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "shipping-promotion-id": { + "type": ["null", "string"] + }, + "delivery-instructions": { + "type": ["null", "string"] + }, + "order-channel": { + "type": ["null", "string"] + }, + "order-channel-instance": { + "type": ["null", "string"] }, - "reportDocumentId": { + "is-business-order": { "type": ["null", "string"] }, - "reportId": { + "purchase-order-number": { "type": ["null", "string"] }, - "dataEndTime": { - "type": ["null", "string"], - "format": "date-time" + "price-designation": { + "type": ["null", "string"] + }, + "buyer-company-name": { + "type": ["null", "string"] }, - "createdTime": { - "type": ["null", "string"], - "format": "date-time" + "licensee-name": { + "type": ["null", "string"] }, - "processingStartTime": { - "type": ["null", "string"], - "format": "date-time" + "license-number": { + "type": ["null", "string"] + }, + "license-state": { + "type": ["null", "string"] }, - "dataStartTime": { - "type": ["null", "string"], - "format": "date-time" + "license-expiration-date": { + "type": ["null", "string"] + }, + "Address-Type": { + "type": ["null", "string"] + }, + "Number-of-items": { + "type": ["null", "string"] + }, + "is-global-express": { + "type": ["null", "string"] + }, + "default-ship-from-address-name": { + "type": ["null", "string"] + }, + "default-ship-from-address-field-1": { + "type": ["null", "string"] + }, + "default-ship-from-address-field-2": { + "type": ["null", "string"] + }, + "default-ship-from-address-field-3": { + "type": ["null", "string"] + }, + "default-ship-from-address-city": { + "type": ["null", "string"] + }, + "default-ship-from-address-state": { + "type": ["null", "string"] + }, + "default-ship-from-address-country": { + "type": ["null", "string"] + }, + "default-ship-from-address-postal-code": { + "type": ["null", "string"] + }, + "actual-ship-from-address-name": { + "type": ["null", "string"] + }, + "actual-ship-from-address-1": { + "type": ["null", "string"] + }, + "actual-ship-from-address-field-2": { + "type": ["null", "string"] + }, + "actual-ship-from-address-field-3": { + "type": ["null", "string"] + }, + "actual-ship-from-address-city": { + "type": ["null", "string"] + }, + "actual-ship-from-address-state": { + "type": ["null", "string"] + }, + "actual-ship-from-address-country": { + "type": ["null", "string"] + }, + "actual-ship-from-address-postal-code": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json new file mode 100644 index 000000000000..d3baf1147640 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json @@ -0,0 +1,77 @@ +{ + "title": "Flat File Open Listings Data", + "description": "Flat File Open Listings Data Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "sku": { + "type": ["null", "string"] + }, + "asin": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "string"] + }, + "Business Price": { + "type": ["null", "string"] + }, + "Quantity Price Type": { + "type": ["null", "string"] + }, + "Quantity Lower Bound 1": { + "type": ["null", "string"] + }, + "Quantity Price 1": { + "type": ["null", "string"] + }, + "Quantity Lower Bound 2": { + "type": ["null", "string"] + }, + "Quantity Price 2": { + "type": ["null", "string"] + }, + "Quantity Lower Bound 3": { + "type": ["null", "string"] + }, + "Quantity Price 3": { + "type": ["null", "string"] + }, + "Quantity Lower Bound 4": { + "type": ["null", "string"] + }, + "Quantity Price 4": { + "type": ["null", "string"] + }, + "Quantity Lower Bound 5": { + "type": ["null", "string"] + }, + "Quantity Price 5": { + "type": ["null", "string"] + }, + "Progressive Price Type": { + "type": ["null", "string"] + }, + "Progressive Lower Bound 1": { + "type": ["null", "string"] + }, + "Progressive Price 1": { + "type": ["null", "string"] + }, + "Progressive Lower Bound 2": { + "type": ["null", "string"] + }, + "Progressive Price 2": { + "type": ["null", "string"] + }, + "Progressive Lower Bound 3": { + "type": ["null", "string"] + }, + "Progressive Price 3": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json index 7482dd295809..b9042d69e2c3 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json @@ -1,46 +1,96 @@ { "title": "Get Merchant Listings Reports", "description": "Get Merchant Listings All Data Reports", - "type": ["null", "object"], + "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "reportType": { + "item-name": { "type": ["null", "string"] }, - "processingEndTime": { + "item-description": { + "type": ["null", "string"] + }, + "listing-id": { + "type": ["null", "string"] + }, + "seller-sku": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "number"] + }, + "quantity": { + "type": ["null", "number"] + }, + "open-date": { "type": ["null", "string"], "format": "date-time" }, - "processingStatus": { + "image-url": { "type": ["null", "string"] }, - "marketplaceIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "item-is-marketplace": { + "type": ["null", "string"] }, - "reportDocumentId": { + "product-id-type": { "type": ["null", "string"] }, - "reportId": { + "zshop-shipping-fee": { "type": ["null", "string"] }, - "dataEndTime": { - "type": ["null", "string"], - "format": "date-time" + "item-note": { + "type": ["null", "string"] }, - "createdTime": { - "type": ["null", "string"], - "format": "date-time" + "item-condition": { + "type": ["null", "string"] }, - "processingStartTime": { - "type": ["null", "string"], - "format": "date-time" + "zshop-category1": { + "type": ["null", "string"] }, - "dataStartTime": { - "type": ["null", "string"], - "format": "date-time" + "zshop-browse-path": { + "type": ["null", "string"] + }, + "zshop-storefront-feature": { + "type": ["null", "string"] + }, + "asin1": { + "type": ["null", "string"] + }, + "asin2": { + "type": ["null", "string"] + }, + "asin3": { + "type": ["null", "string"] + }, + "will-ship-internationally": { + "type": ["null", "string"] + }, + "expedited-shipping": { + "type": ["null", "string"] + }, + "zshop-boldface": { + "type": ["null", "string"] + }, + "product-id": { + "type": ["null", "string"] + }, + "bid-for-featured-placement": { + "type": ["null", "string"] + }, + "add-delete": { + "type": ["null", "string"] + }, + "pending-quantity": { + "type": ["null", "number"] + }, + "fulfillment-channel": { + "type": ["null", "string"] + }, + "merchant-shipping-group": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT.json new file mode 100644 index 000000000000..5b48d11d5e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT.json @@ -0,0 +1,20 @@ +{ + "title": "Vendor Inventory Health and Planning Data", + "description": "Vendor Inventory Health and Planning Data Reports", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "seller-sku": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "number"] + }, + "product ID": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json index 3617727f0704..6eef79420d93 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json @@ -1,7 +1,7 @@ { "title": "Orders", "description": "All orders that were updated after a specified date", - "type": ["null", "object"], + "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "seller_id": { diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json new file mode 100644 index 000000000000..e7a56df4734e --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json @@ -0,0 +1,242 @@ +{ + "title": "Vendor Direct Fulfillment Shipping", + "description": "Vendor Direct Fulfillment Shipping", + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "purchaseOrderNumber": { + "type": ["null", "string"] + }, + "sellingParty": { + "type": ["null", "object"], + "properties": { + "partyId": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxRegistrationDetails": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "taxRegistrationType": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + }, + "taxRegistrationAddress": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxRegistrationMessages": { + "type": ["null", "string"] + } + } + } + } + } + }, + "shipFromParty": { + "type": ["null", "object"], + "properties": { + "partyId": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxRegistrationDetails": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "taxRegistrationType": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + }, + "taxRegistrationAddress": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "addressLine1": { + "type": ["null", "string"] + }, + "addressLine2": { + "type": ["null", "string"] + }, + "addressLine3": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "county": { + "type": ["null", "string"] + }, + "district": { + "type": ["null", "string"] + }, + "stateOrRegion": { + "type": ["null", "string"] + }, + "postalCode": { + "type": ["null", "string"] + }, + "countryCode": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "taxRegistrationMessages": { + "type": ["null", "string"] + } + } + } + } + } + }, + "labelFormat": { + "type": ["null", "string"] + }, + "labelData": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "packageIdentifier": { + "type": ["null", "string"] + }, + "trackingNumber": { + "type": ["null", "string"] + }, + "shipMethod": { + "type": ["null", "string"] + }, + "shipMethodName": { + "type": ["null", "string"] + }, + "content": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index dd214f9ac806..788bee2c65dd 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -25,15 +25,27 @@ from typing import Any, List, Mapping, Tuple import boto3 +import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification, SyncMode +from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from pydantic import Field from pydantic.main import BaseModel from source_amazon_seller_partner.auth import AWSAuthenticator, AWSSignature from source_amazon_seller_partner.constants import AWSEnvironment, AWSRegion, get_marketplaces -from source_amazon_seller_partner.streams import FbaInventoryReports, FlatFileOrdersReports, MerchantListingsReports, Orders +from source_amazon_seller_partner.streams import ( + FbaInventoryReports, + FbaOrdersReports, + FbaShipmentsReports, + FlatFileOpenListingsReports, + FlatFileOrdersReports, + FulfilledShipmentsReports, + MerchantListingsReports, + Orders, + VendorDirectFulfillmentShipping, + VendorInventoryHealthReports, +) class ConnectorConfig(BaseModel): @@ -60,8 +72,8 @@ class Config: class SourceAmazonSellerPartner(AbstractSource): - def _get_stream_kwargs(self, config: ConnectorConfig): - self.endpoint, self.marketplace_id, self.region = get_marketplaces(config.aws_environment)[config.region] + def _get_stream_kwargs(self, config: ConnectorConfig) -> Mapping[str, Any]: + endpoint, marketplace_id, region = get_marketplaces(config.aws_environment)[config.region] boto3_client = boto3.client("sts", aws_access_key_id=config.aws_access_key, aws_secret_access_key=config.aws_secret_key) role = boto3_client.assume_role(RoleArn=config.role_arn, RoleSessionName="guid") @@ -71,32 +83,46 @@ def _get_stream_kwargs(self, config: ConnectorConfig): aws_access_key_id=role_creds.get("AccessKeyId"), aws_secret_access_key=role_creds.get("SecretAccessKey"), aws_session_token=role_creds.get("SessionToken"), - region=self.region, + region=region, ) auth = AWSAuthenticator( token_refresh_endpoint="https://api.amazon.com/auth/o2/token", client_secret=config.lwa_client_secret, client_id=config.lwa_app_id, refresh_token=config.refresh_token, - host=self.endpoint.replace("https://", ""), + host=endpoint.replace("https://", ""), ) stream_kwargs = { - "url_base": self.endpoint, + "url_base": endpoint, "authenticator": auth, "aws_signature": aws_signature, "replication_start_date": config.replication_start_date, + "marketplace_ids": [marketplace_id], } return stream_kwargs def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + """ + Check connection to Amazon SP API by requesting the list of reports as this endpoint should be available for any config. + Validate if response has the expected error code and body. + Show error message in case of request exception or unexpected response. + """ + + error_msg = "Unable to connect to Amazon Seller API with the provided credentials - {error}" try: config = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK stream_kwargs = self._get_stream_kwargs(config) - merchant_listings_reports_gen = MerchantListingsReports(**stream_kwargs).read_records(sync_mode=SyncMode.full_refresh) - next(merchant_listings_reports_gen) - return True, None + + reports_res = requests.get( + url=f"{stream_kwargs['url_base']}{MerchantListingsReports.path_prefix}/reports", + headers={**stream_kwargs["authenticator"].get_auth_header(), "content-type": "application/json"}, + params={"reportTypes": MerchantListingsReports.name}, + auth=stream_kwargs["aws_signature"], + ) + connected = reports_res.status_code == 200 and reports_res.json().get("payload") + return connected, None if connected else error_msg.format(error=reports_res.json()) except Exception as error: - return False, f"Unable to connect to Amazon Seller API with the provided credentials - {repr(error)}" + return False, error_msg.format(error=repr(error)) def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ @@ -106,10 +132,16 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: stream_kwargs = self._get_stream_kwargs(config) return [ - MerchantListingsReports(**stream_kwargs), - FlatFileOrdersReports(**stream_kwargs), FbaInventoryReports(**stream_kwargs), - Orders(marketplace_ids=[self.marketplace_id], **stream_kwargs), + FbaOrdersReports(**stream_kwargs), + FbaShipmentsReports(**stream_kwargs), + FlatFileOpenListingsReports(**stream_kwargs), + FlatFileOrdersReports(**stream_kwargs), + FulfilledShipmentsReports(**stream_kwargs), + MerchantListingsReports(**stream_kwargs), + VendorDirectFulfillmentShipping(**stream_kwargs), + VendorInventoryHealthReports(**stream_kwargs), + Orders(**stream_kwargs), ] def spec(self, *args, **kwargs) -> ConnectorSpecification: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index cce7aa6c3deb..bb3c8c1863c9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -22,32 +22,61 @@ # SOFTWARE. # +import base64 +import csv +import json as json_lib +import time +import zlib from abc import ABC, abstractmethod +from io import StringIO from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +import pendulum import requests +from airbyte_cdk.entrypoint import logger +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, NoAuth +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException +from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS +from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler +from Crypto.Cipher import AES from source_amazon_seller_partner.auth import AWSSignature REPORTS_API_VERSION = "2020-09-04" ORDERS_API_VERSION = "v0" +VENDORS_API_VERSION = "v1" + +REPORTS_MAX_WAIT_SECONDS = 50 class AmazonSPStream(HttpStream, ABC): - page_size = 100 data_field = "payload" - def __init__(self, url_base: str, aws_signature: AWSSignature, replication_start_date: str, *args, **kwargs): + def __init__( + self, url_base: str, aws_signature: AWSSignature, replication_start_date: str, marketplace_ids: List[str], *args, **kwargs + ): super().__init__(*args, **kwargs) self._url_base = url_base - self._aws_signature = aws_signature self._replication_start_date = replication_start_date + self.marketplace_ids = marketplace_ids + self._session.auth = aws_signature @property def url_base(self) -> str: return self._url_base + def request_headers(self, *args, **kwargs) -> Mapping[str, Any]: + return {"content-type": "application/json"} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + +class IncrementalAmazonSPStream(AmazonSPStream, ABC): + page_size = 100 + @property @abstractmethod def replication_start_date_field(self) -> str: @@ -75,7 +104,7 @@ def request_params( return dict(next_page_token) params = {self.replication_start_date_field: self._replication_start_date, self.page_size_field: self.page_size} - if self._replication_start_date: + if self._replication_start_date and self.cursor_field: start_date = max(stream_state.get(self.cursor_field, self._replication_start_date), self._replication_start_date) params.update({self.replication_start_date_field: start_date}) return params @@ -102,55 +131,242 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} return {self.cursor_field: latest_benchmark} - def _create_prepared_request( - self, path: str, headers: Mapping = None, params: Mapping = None, json: Any = None, data: Any = None - ) -> requests.PreparedRequest: - """ - Override to prepare request for AWS API. - AWS signature flow require prepared request to correctly generate `authorization` header. - Add `auth` arg to sign all the requests with AWS signature. - """ - return self._session.prepare_request( - requests.Request(method=self.http_method, url=self.url_base + path, headers=headers, params=params, auth=self._aws_signature) - ) +class ReportsAmazonSPStream(Stream, ABC): + """ + API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reports_2020-09-04.md + API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/reports-api-model/reports_2020-09-04.json - def request_headers(self, *args, **kwargs) -> Mapping[str, Any]: + Report streams are intended to work as following: + - create a new report; + - retrieve the report; + - retry the retrieval if the report is still not fully processed; + - retrieve the report document (if report processing status is `DONE`); + - decrypt the report document (if report processing status is `DONE`); + - yield the report document (if report processing status is `DONE`) + """ + + primary_key = None + path_prefix = f"/reports/{REPORTS_API_VERSION}" + sleep_seconds = 30 + data_field = "payload" + + def __init__( + self, + url_base: str, + aws_signature: AWSSignature, + replication_start_date: str, + marketplace_ids: List[str], + authenticator: HttpAuthenticator = NoAuth(), + ): + self._authenticator = authenticator + self._session = requests.Session() + self._url_base = url_base + self._session.auth = aws_signature + self._replication_start_date = replication_start_date + self.marketplace_ids = marketplace_ids + + @property + def url_base(self) -> str: + return self._url_base + + @property + def authenticator(self) -> HttpAuthenticator: + return self._authenticator + + def request_params(self) -> MutableMapping[str, Any]: + return {"MarketplaceIds": ",".join(self.marketplace_ids)} + + def request_headers(self) -> Mapping[str, Any]: return {"content-type": "application/json"} + def path(self, document_id: str) -> str: + return f"{self.path_prefix}/documents/{document_id}" -class ReportsBase(AmazonSPStream, ABC): - primary_key = "reportId" - cursor_field = "createdTime" - replication_start_date_field = "createdSince" - next_page_token_field = "nextToken" - page_size_field = "pageSize" + def should_retry(self, response: requests.Response) -> bool: + return response.status_code == 429 or 500 <= response.status_code < 600 - def path(self, **kwargs): - return f"/reports/{REPORTS_API_VERSION}/reports" + @default_backoff_handler(max_tries=5, factor=5) + def _send_request(self, request: requests.PreparedRequest) -> requests.Response: + response: requests.Response = self._session.send(request) + if self.should_retry(response): + raise DefaultBackoffException(request=request, response=response) + else: + response.raise_for_status() + return response - def request_params( - self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, next_page_token, **kwargs) - if not next_page_token: - params.update({"reportTypes": self.name}) - return params + def _create_prepared_request( + self, path: str, http_method: str = "GET", headers: Mapping = None, params: Mapping = None, json: Any = None, data: Any = None + ) -> requests.PreparedRequest: + """ + Override to make http_method configurable per method call + """ + args = {"method": http_method, "url": self.url_base + path, "headers": headers, "params": params} + if http_method.upper() in BODY_REQUEST_METHODS: + if json and data: + raise RequestBodyException( + "At the same time only one of the 'request_body_data' and 'request_body_json' functions can return data" + ) + elif json: + args["json"] = json + elif data: + args["data"] = data + + return self._session.prepare_request(requests.Request(**args)) + + def _create_report(self) -> Mapping[str, Any]: + request_headers = self.request_headers() + replication_start_date = max(pendulum.parse(self._replication_start_date), pendulum.now("utc").subtract(days=90)) + report_data = { + "reportType": self.name, + "marketplaceIds": self.marketplace_ids, + "createdSince": replication_start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + } + create_report_request = self._create_prepared_request( + http_method="POST", + path=f"{self.path_prefix}/reports", + headers=dict(request_headers, **self.authenticator.get_auth_header()), + data=json_lib.dumps(report_data), + ) + report_response = self._send_request(create_report_request) + return report_response.json()[self.data_field] + + def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: + request_headers = self.request_headers() + retrieve_report_request = self._create_prepared_request( + path=f"{self.path_prefix}/reports/{report_id}", + headers=dict(request_headers, **self.authenticator.get_auth_header()), + ) + retrieve_report_response = self._send_request(retrieve_report_request) + report_payload = retrieve_report_response.json().get(self.data_field, {}) + return report_payload + + @staticmethod + def decrypt_aes(content, key, iv): + key = base64.b64decode(key) + iv = base64.b64decode(iv) + decrypter = AES.new(key, AES.MODE_CBC, iv) + decrypted = decrypter.decrypt(content) + padding_bytes = decrypted[-1] + return decrypted[:-padding_bytes] + + def decrypt_report_document(self, url, initialization_vector, key, encryption_standard, payload): + """ + Decrypts and unpacks a report document, currently AES encryption is implemented + """ + if encryption_standard == "AES": + decrypted = self.decrypt_aes(requests.get(url).content, key, initialization_vector) + if "compressionAlgorithm" in payload: + return zlib.decompress(bytearray(decrypted), 15 + 32).decode("iso-8859-1") + return decrypted.decode("iso-8859-1") + raise Exception([{"message": "Only AES decryption is implemented."}]) + + def parse_response(self, response: requests.Response) -> Iterable[Mapping]: + payload = response.json().get(self.data_field, {}) + document = self.decrypt_report_document( + payload.get("url"), + payload.get("encryptionDetails", {}).get("initializationVector"), + payload.get("encryptionDetails", {}).get("key"), + payload.get("encryptionDetails", {}).get("standard"), + payload, + ) + document_records = csv.DictReader(StringIO(document), delimiter="\t") + yield from document_records -class MerchantListingsReports(ReportsBase): + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + Create and retrieve the report. + Decrypt and parse the report is its fully proceed, then yield the report document records. + """ + report_payload = {} + is_processed = False + is_done = False + start_time = pendulum.now("utc") + seconds_waited = 0 + report_id = self._create_report()["reportId"] + + # create and retrieve the report + while not is_processed and seconds_waited < REPORTS_MAX_WAIT_SECONDS: + report_payload = self._retrieve_report(report_id=report_id) + seconds_waited = (pendulum.now("utc") - start_time).seconds + is_processed = report_payload.get("processingStatus") not in ["IN_QUEUE", "IN_PROGRESS"] + is_done = report_payload.get("processingStatus") == "DONE" + time.sleep(self.sleep_seconds) + + if is_done: + # retrieve and decrypt the report document + document_id = report_payload["reportDocumentId"] + request_headers = self.request_headers() + request = self._create_prepared_request( + path=self.path(document_id=document_id), + headers=dict(request_headers, **self.authenticator.get_auth_header()), + params=self.request_params(), + ) + response = self._send_request(request) + yield from self.parse_response(response) + else: + logger.warn(f"There are no report document related in stream `{self.name}`. Report body {report_payload}") + + +class MerchantListingsReports(ReportsAmazonSPStream): name = "GET_MERCHANT_LISTINGS_ALL_DATA" -class FlatFileOrdersReports(ReportsBase): +class FlatFileOrdersReports(ReportsAmazonSPStream): + """ + Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=201648780 + """ + name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" -class FbaInventoryReports(ReportsBase): +class FbaInventoryReports(ReportsAmazonSPStream): + """ + Field definitions: https://sellercentral.amazon.com/gp/help/200740930 + """ + name = "GET_FBA_INVENTORY_AGED_DATA" -class Orders(AmazonSPStream): +class FulfilledShipmentsReports(ReportsAmazonSPStream): + """ + Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200453120 + """ + + name = "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL" + + +class FlatFileOpenListingsReports(ReportsAmazonSPStream): + name = "GET_FLAT_FILE_OPEN_LISTINGS_DATA" + + +class FbaOrdersReports(ReportsAmazonSPStream): + """ + Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989110 + """ + + name = "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA" + + +class FbaShipmentsReports(ReportsAmazonSPStream): + """ + Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100 + """ + + name = "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA" + + +class VendorInventoryHealthReports(ReportsAmazonSPStream): + name = "GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT" + + +class Orders(IncrementalAmazonSPStream): + """ + API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/orders-api/ordersV0.md + API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/orders-api-model/ordersV0.json + """ + name = "Orders" primary_key = "AmazonOrderId" cursor_field = "LastUpdateDate" @@ -158,23 +374,54 @@ class Orders(AmazonSPStream): next_page_token_field = "NextToken" page_size_field = "MaxResultsPerPage" - def __init__(self, marketplace_ids: List[str], **kwargs): - super().__init__(**kwargs) - self.marketplace_ids = marketplace_ids - - def path(self, **kwargs): + def path(self, **kwargs) -> str: return f"/orders/{ORDERS_API_VERSION}/orders" def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, next_page_token, **kwargs) + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) if not next_page_token: params.update({"MarketplaceIds": ",".join(self.marketplace_ids)}) return params def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ yield from response.json().get(self.data_field, {}).get(self.name, []) + + +class VendorDirectFulfillmentShipping(AmazonSPStream): + """ + API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/vendor-direct-fulfillment-shipping-api/vendorDirectFulfillmentShippingV1.md + API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/vendor-direct-fulfillment-shipping-api-model/vendorDirectFulfillmentShippingV1.json + + Returns a list of shipping labels created during the time frame that you specify. + Both createdAfter and createdBefore parameters required to select the time frame. + The date range to search must not be more than 7 days. + """ + + name = "VendorDirectFulfillmentShipping" + primary_key = [["labelData", "packageIdentifier"]] + replication_start_date_field = "createdAfter" + next_page_token_field = "nextToken" + page_size_field = "limit" + time_format = "%Y-%m-%dT%H:%M:%SZ" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.replication_start_date_field = max( + pendulum.parse(self._replication_start_date), pendulum.now("utc").subtract(days=7, hours=1) + ).strftime(self.time_format) + + def path(self, **kwargs) -> str: + return f"/vendor/directFulfillment/shipping/{VENDORS_API_VERSION}/shippingLabels" + + def request_params( + self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + if not next_page_token: + params.update({"createdBefore": pendulum.now("utc").strftime(self.time_format)}) + return params + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + yield from response.json().get(self.data_field, {}).get("shippingLabels", []) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_repots_streams_rate_limits.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_repots_streams_rate_limits.py new file mode 100644 index 000000000000..293698fdc831 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_repots_streams_rate_limits.py @@ -0,0 +1,84 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import time + +import pytest +import requests +from airbyte_cdk.sources.streams.http.auth import NoAuth +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException +from source_amazon_seller_partner.auth import AWSSignature +from source_amazon_seller_partner.streams import MerchantListingsReports + + +@pytest.fixture +def reports_stream(): + aws_signature = AWSSignature( + service="execute-api", + aws_access_key_id="AccessKeyId", + aws_secret_access_key="SecretAccessKey", + aws_session_token="SessionToken", + region="US", + ) + stream = MerchantListingsReports( + url_base="https://test.url", + aws_signature=aws_signature, + replication_start_date="2017-01-25T00:00:00Z", + marketplace_ids=["id"], + authenticator=NoAuth(), + ) + return stream + + +def test_reports_stream_should_retry(mocker, reports_stream): + response = requests.Response() + response.status_code = 429 + mocker.patch.object(requests.Session, "send", return_value=response) + should_retry = reports_stream.should_retry(response=response) + + assert should_retry is True + + +def test_reports_stream_send_request(mocker, reports_stream): + response = requests.Response() + response.status_code = 200 + mocker.patch.object(requests.Session, "send", return_value=response) + + assert response == reports_stream._send_request(request=requests.PreparedRequest()) + + +def test_reports_stream_send_request_backoff_exception(mocker, caplog, reports_stream): + response = requests.Response() + response.status_code = 429 + mocker.patch.object(requests.Session, "send", return_value=response) + mocker.patch.object(time, "sleep", return_value=None) + + with pytest.raises(DefaultBackoffException): + reports_stream._send_request(request=requests.PreparedRequest()) + + assert "Backing off _send_request(...) for 5.0s" in caplog.text + assert "Backing off _send_request(...) for 10.0s" in caplog.text + assert "Backing off _send_request(...) for 20.0s" in caplog.text + assert "Backing off _send_request(...) for 40.0s" in caplog.text + assert "Giving up _send_request(...) after 5 tries" in caplog.text diff --git a/airbyte-integrations/connectors/source-amplitude/CHANGELOG.md b/airbyte-integrations/connectors/source-amplitude/CHANGELOG.md deleted file mode 100644 index 45fcf432379b..000000000000 --- a/airbyte-integrations/connectors/source-amplitude/CHANGELOG.md +++ /dev/null @@ -1,4 +0,0 @@ -# Changelog - -## 0.1.0 -Source implementation. diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile index 8f2878b85790..354ef70a04c3 100644 --- a/airbyte-integrations/connectors/source-amplitude/Dockerfile +++ b/airbyte-integrations/connectors/source-amplitude/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml index 3fe739130440..dcc269fb78db 100644 --- a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml @@ -16,9 +16,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_without_events.json" # Unable to use 'state_path' because Amplitude returns an error when specifying a date in the future. - # state_path: "integration_tests/abnormal_state.json" - cursor_paths: - active_users: [ "date" ] + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_without_events.json" diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json index 786041dfa7b4..5a9f565cb48d 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json @@ -3,60 +3,7 @@ { "stream": { "name": "cohorts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "appId": { - "type": ["null", "integer"] - }, - "archived": { - "type": ["null", "boolean"] - }, - "definition": { - "type": ["null", "object"] - }, - "description": { - "type": ["null", "string"] - }, - "finished": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "owners": { - "type": ["null", "object"] - }, - "published": { - "type": ["null", "boolean"] - }, - "size": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "lastMod": { - "type": ["null", "integer"] - }, - "lastComputed": { - "type": ["null", "integer"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "is_predictive": { - "type": ["null", "boolean"] - }, - "is_official_content": { - "type": ["null", "boolean"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, @@ -66,25 +13,7 @@ { "stream": { "name": "annotations", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "date": { - "type": ["null", "string"], - "format": "date-time" - }, - "details": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "label": { - "type": ["null", "string"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, @@ -94,166 +23,7 @@ { "stream": { "name": "events", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "server_received_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "app": { - "type": ["null", "integer"] - }, - "device_carrier": { - "type": ["null", "string"] - }, - "$schema": { - "type": ["null", "integer"] - }, - "city": { - "type": ["null", "string"] - }, - "user_id": { - "type": ["null", "string"] - }, - "uuid": { - "type": ["null", "string"] - }, - "event_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "platform": { - "type": ["null", "string"] - }, - "os_version": { - "type": ["null", "string"] - }, - "amplitude_id": { - "type": ["null", "integer"] - }, - "processed_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "user_creation_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "version_name": { - "type": ["null", "string"] - }, - "ip_address": { - "type": ["null", "string"] - }, - "paying": { - "type": ["null", "boolean"] - }, - "dma": { - "type": ["null", "string"] - }, - "group_properties": { - "type": ["null", "object"] - }, - "user_properties": { - "type": ["null", "object"] - }, - "client_upload_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "$insert_id": { - "type": ["null", "string"] - }, - "event_type": { - "type": ["null", "string"] - }, - "library": { - "type": ["null", "string"] - }, - "amplitude_attribution_ids": { - "type": ["null", "string"] - }, - "device_type": { - "type": ["null", "string"] - }, - "device_manufacturer": { - "type": ["null", "string"] - }, - "start_version": { - "type": ["null", "string"] - }, - "location_lng": { - "type": ["null", "number"] - }, - "server_upload_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "event_id": { - "type": ["null", "integer"] - }, - "location_lat": { - "type": ["null", "number"] - }, - "os_name": { - "type": ["null", "string"] - }, - "amplitude_event_type": { - "type": ["null", "string"] - }, - "device_brand": { - "type": ["null", "string"] - }, - "groups": { - "type": ["null", "object"] - }, - "event_properties": { - "type": ["null", "object"] - }, - "data": { - "type": ["null", "object"] - }, - "device_id": { - "type": ["null", "string"] - }, - "language": { - "type": ["null", "string"] - }, - "device_model": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "is_attribution_event": { - "type": ["null", "boolean"] - }, - "adid": { - "type": ["null", "string"] - }, - "session_id": { - "type": ["null", "number"] - }, - "device_family": { - "type": ["null", "string"] - }, - "sample_rate": { - "type": ["null"] - }, - "idfa": { - "type": ["null", "string"] - }, - "client_event_time": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["event_time"], @@ -266,19 +36,7 @@ { "stream": { "name": "active_users", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "date": { - "type": ["null", "string"], - "format": "date-time" - }, - "statistics": { - "type": ["null", "object"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], @@ -291,19 +49,7 @@ { "stream": { "name": "average_session_length", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "date": { - "type": ["null", "string"], - "format": "date-time" - }, - "length": { - "type": ["null", "number"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["date"], diff --git a/airbyte-integrations/connectors/source-amplitude/requirements.txt b/airbyte-integrations/connectors/source-amplitude/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json index f8707f899b84..fe21f00a5124 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json @@ -4,7 +4,7 @@ "properties": { "date": { "type": ["null", "string"], - "format": "date-time" + "format": "date" }, "statistics": { "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json index 6d52f4e13510..7f0e80192f55 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json @@ -4,7 +4,7 @@ "properties": { "date": { "type": ["null", "string"], - "format": "date-time" + "format": "date" }, "length": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json index 71bd4520bcb5..2f29eb48c231 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json @@ -24,7 +24,10 @@ "type": ["null", "string"] }, "owners": { - "type": ["null", "object"] + "type": ["null", "array"], + "items": { + "type": "string" + } }, "published": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json index 6638e22de4f5..dcc2aa12b5fd 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json @@ -147,7 +147,7 @@ "type": ["null", "string"] }, "sample_rate": { - "type": ["null"] + "type": ["null", "string", "number"] }, "idfa": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bigquery/CHANGELOG.md b/airbyte-integrations/connectors/source-bigquery/CHANGELOG.md new file mode 100644 index 000000000000..ff77c983a393 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## 0.1.2 +Fix for NPE when optional `dataset_id` not provided in configuration. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-bigquery/Dockerfile b/airbyte-integrations/connectors/source-bigquery/Dockerfile index 9cdae4cb9f47..77d6d5f84cab 100644 --- a/airbyte-integrations/connectors/source-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/source-bigquery/Dockerfile @@ -9,5 +9,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/source-bigquery \ No newline at end of file +LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.name=airbyte/source-bigquery diff --git a/airbyte-integrations/connectors/source-bigquery/build.gradle b/airbyte-integrations/connectors/source-bigquery/build.gradle index 6c2da5248df7..f0c226ac4618 100644 --- a/airbyte-integrations/connectors/source-bigquery/build.gradle +++ b/airbyte-integrations/connectors/source-bigquery/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.bigquery.BigQuerySource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java index 8d53f39de3ae..3bc10b007518 100644 --- a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java +++ b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java @@ -122,12 +122,11 @@ protected List>> discoverInternal(Big @Override protected List>> discoverInternal(BigQueryDatabase database, String schema) { String projectId = dbConfig.get(CONFIG_PROJECT_ID).asText(); - String datasetId = getConfigDatasetId(database); List tables = (isDatasetConfigured(database) ? database.getDatasetTables(getConfigDatasetId(database)) : database.getProjectTables(projectId)); List>> result = new ArrayList<>(); tables.stream().map(table -> TableInfo.>builder() - .nameSpace(datasetId) + .nameSpace(table.getTableId().getDataset()) .name(table.getTableId().getTable()) .fields(Objects.requireNonNull(table.getDefinition().getSchema()).getFields().stream() .map(f -> { @@ -177,11 +176,12 @@ private AutoCloseableIterator queryTableWithParams(BigQueryDatabase da } private boolean isDatasetConfigured(SqlDatabase database) { - return database.getSourceConfig().hasNonNull(CONFIG_DATASET_ID); + JsonNode config = database.getSourceConfig(); + return config.hasNonNull(CONFIG_DATASET_ID) ? !config.get(CONFIG_DATASET_ID).asText().isEmpty() : false; } private String getConfigDatasetId(SqlDatabase database) { - return (isDatasetConfigured(database) ? database.getSourceConfig().get(CONFIG_DATASET_ID).asText() : null); + return (isDatasetConfigured(database) ? database.getSourceConfig().get(CONFIG_DATASET_ID).asText() : ""); } public static void main(String[] args) throws Exception { diff --git a/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceTest.java b/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceTest.java new file mode 100644 index 000000000000..77d0dfc42188 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/src/test-integration/java/io/airbyte/integrations/source/bigquery/BigQuerySourceTest.java @@ -0,0 +1,125 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.bigquery; + +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_CREDS; +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_DATASET_ID; +import static io.airbyte.integrations.source.bigquery.BigQuerySource.CONFIG_PROJECT_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.db.bigquery.BigQueryDatabase; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.SQLException; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class BigQuerySourceTest { + + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); + private static final String STREAM_NAME = "id_and_name"; + + private BigQueryDatabase database; + private Dataset dataset; + private JsonNode config; + + @BeforeEach + void setUp() throws IOException, SQLException { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a big query credentials file. By default {module-root}/" + CREDENTIALS_PATH + + ". Override by setting setting path with the CREDENTIALS_PATH constant."); + } + + final String credentialsJsonString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + + final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString); + final String projectId = credentialsJson.get(CONFIG_PROJECT_ID).asText(); + final String datasetLocation = "US"; + + final String datasetId = Strings.addRandomSuffix("airbyte_tests", "_", 8); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(CONFIG_PROJECT_ID, projectId) + .put(CONFIG_CREDS, credentialsJsonString) + .put(CONFIG_DATASET_ID, datasetId) + .build()); + + database = new BigQueryDatabase(config.get(CONFIG_PROJECT_ID).asText(), credentialsJsonString); + + final DatasetInfo datasetInfo = + DatasetInfo.newBuilder(config.get(CONFIG_DATASET_ID).asText()).setLocation(datasetLocation).build(); + dataset = database.getBigQuery().create(datasetInfo); + + database.execute( + "CREATE TABLE " + datasetId + + ".id_and_name(id INT64, array_val ARRAY>>, object_val STRUCT>, value_str2 string>);"); + database.execute( + "INSERT INTO " + datasetId + + ".id_and_name (id, array_val, object_val) VALUES " + + "(1, [STRUCT('test1_1', STRUCT('struct1_1')), STRUCT('test1_2', STRUCT('struct1_2'))], STRUCT([STRUCT('value1_1'), STRUCT('value1_2')], 'test1_1')), " + + "(2, [STRUCT('test2_1', STRUCT('struct2_1')), STRUCT('test2_2', STRUCT('struct2_2'))], STRUCT([STRUCT('value2_1'), STRUCT('value2_2')], 'test2_1')), " + + "(3, [STRUCT('test3_1', STRUCT('struct3_1')), STRUCT('test3_2', STRUCT('struct3_2'))], STRUCT([STRUCT('value3_1'), STRUCT('value3_2')], 'test3_1'));"); + } + + @AfterEach + void tearDown() { + database.cleanDataSet(dataset.getDatasetId().getDataset()); + } + + @Test + public void testReadSuccess() throws Exception { + final List actualMessages = MoreIterators.toList(new BigQuerySource().read(config, getConfiguredCatalog(), null)); + + assertNotNull(actualMessages); + assertEquals(3, actualMessages.size()); + } + + private ConfiguredAirbyteCatalog getConfiguredCatalog() { + return CatalogHelpers.createConfiguredAirbyteCatalog( + STREAM_NAME, + config.get(CONFIG_DATASET_ID).asText(), + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("array_val", JsonSchemaPrimitive.ARRAY), + Field.of("object_val", JsonSchemaPrimitive.OBJECT)); + } + +} diff --git a/airbyte-integrations/connectors/source-bigquery/src/test/java/io/airbyte/integrations/source/bigquery/BigQuerySourceTest.java b/airbyte-integrations/connectors/source-bigquery/src/test/java/io/airbyte/integrations/source/bigquery/BigQuerySourceTest.java new file mode 100644 index 000000000000..5f74ac2143df --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/src/test/java/io/airbyte/integrations/source/bigquery/BigQuerySourceTest.java @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.bigquery; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import java.io.IOException; +import org.junit.jupiter.api.Test; + +class BigQuerySourceTest { + + @Test + public void testEmptyDatasetIdInConfig() throws IOException { + JsonNode configJson = Jsons.deserialize(MoreResources.readResource("test_config_empty_datasetid.json")); + JsonNode dbConfig = new BigQuerySource().toDatabaseConfig(configJson); + assertTrue(dbConfig.get(BigQuerySource.CONFIG_DATASET_ID).isEmpty()); + } + + @Test + public void testConfig() throws IOException { + JsonNode configJson = Jsons.deserialize(MoreResources.readResource("test_config.json")); + JsonNode dbConfig = new BigQuerySource().toDatabaseConfig(configJson); + assertEquals("dataset", dbConfig.get(BigQuerySource.CONFIG_DATASET_ID).asText()); + assertEquals("project", dbConfig.get(BigQuerySource.CONFIG_PROJECT_ID).asText()); + assertEquals("credentials", dbConfig.get(BigQuerySource.CONFIG_CREDS).asText()); + } + +} diff --git a/airbyte-integrations/connectors/source-bigquery/src/test/resources/test_config.json b/airbyte-integrations/connectors/source-bigquery/src/test/resources/test_config.json new file mode 100644 index 000000000000..3086a588fab5 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/src/test/resources/test_config.json @@ -0,0 +1,5 @@ +{ + "dataset_id": "dataset", + "project_id": "project", + "credentials_json": "credentials" +} diff --git a/airbyte-integrations/connectors/source-bigquery/src/test/resources/test_config_empty_datasetid.json b/airbyte-integrations/connectors/source-bigquery/src/test/resources/test_config_empty_datasetid.json new file mode 100644 index 000000000000..dcec1604b810 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigquery/src/test/resources/test_config_empty_datasetid.json @@ -0,0 +1,5 @@ +{ + "dataset_id": "", + "project_id": "xxxx", + "credentials_json": "xxxx" +} diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index 88ea836e54c5..a32ff5bb98be 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.clickhouse.ClickHouseSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-cockroachdb/build.gradle b/airbyte-integrations/connectors/source-cockroachdb/build.gradle index c9f27bb7ac38..342895cc0516 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index 85640c75e3cc..5b1ea3f4d56b 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.db2.Db2Source' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile index 60e4d1628891..ca27080bf614 100644 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ b/airbyte-integrations/connectors/source-github/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index 4761883b4dec..bb0bdb411995 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -19,7 +19,7 @@ tests: cursor_paths: comments: ["airbytehq/integration-test", "updated_at"] commit_comments: ["airbytehq/integration-test", "updated_at"] - commits: ["airbytehq/integration-test", "created_at"] + commits: ["airbytehq/integration-test", "master", "created_at"] events: ["airbytehq/integration-test", "created_at"] issue_events: ["airbytehq/integration-test", "created_at"] issue_milestones: ["airbytehq/integration-test", "updated_at"] diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json b/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json index 46e32ab7a7f4..dd1f6c27cc0a 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/commits.json @@ -96,11 +96,11 @@ } } }, - "author_id": { - "type": ["null", "integer"] + "author": { + "$ref": "user.json" }, - "committer_id": { - "type": ["null", "integer"] + "committer": { + "$ref": "user.json" }, "parents": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/events.json index fca685e48023..79ac9cfc0cc3 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/events.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/events.json @@ -15,14 +15,42 @@ "type": ["null", "object"], "properties": {} }, - "repo_id": { - "type": ["null", "integer"] - }, - "actor_id": { - "type": ["null", "integer"] - }, - "org_id": { - "type": ["null", "integer"] + "repo": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "actor": { + "$ref": "user.json" + }, + "org": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "login": { + "type": ["null", "string"] + }, + "gravatar_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + } + } }, "created_at": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json index 8a0bb5731559..8d7e3184514b 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_events.json @@ -14,8 +14,8 @@ "url": { "type": ["null", "string"] }, - "actor_id": { - "type": ["null", "integer"] + "actor": { + "$ref": "user.json" }, "event": { "type": ["null", "string"] @@ -30,8 +30,49 @@ "type": ["null", "string"], "format": "date-time" }, - "issue_id": { - "type": ["null", "integer"] + "issue": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "repository_url": { + "type": ["null", "string"] + }, + "labels_url": { + "type": ["null", "string"] + }, + "comments_url": { + "type": ["null", "string"] + }, + "events_url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "user": { + "$ref": "user.json" + } + } } } } diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json index 6de25d9189fe..9e0eed332b9c 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_milestones.json @@ -32,8 +32,8 @@ "description": { "type": ["null", "string"] }, - "creator_id": { - "type": ["null", "integer"] + "creator": { + "$ref": "user.json" }, "open_issues": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json index a3680341fb6a..bd723d579abb 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issues.json @@ -50,20 +50,97 @@ "labels": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + } + } } }, - "assignee_id": { - "type": ["null", "integer"] + "assignee": { + "$ref": "user.json" }, "assignees": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "$ref": "user.json" } }, - "milestone_id": { - "type": ["null", "integer"] + "milestone": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "labels_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "creator": { + "$ref": "user.json" + }, + "open_issues": { + "type": ["null", "integer"] + }, + "closed_issues": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "due_on": { + "type": ["null", "string"], + "format": "date-time" + } + } }, "locked": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json b/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json index 3bd9053de12e..de1f384c3e8d 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/projects.json @@ -35,8 +35,8 @@ "state": { "type": ["null", "string"] }, - "creator_id": { - "type": ["null", "integer"] + "creator": { + "$ref": "user.json" }, "created_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json index 9c5f16e7da57..f64a7ea03c41 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/pull_requests.json @@ -62,11 +62,88 @@ "labels": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + } + } } }, - "milestone_id": { - "type": ["null", "integer"] + "milestone": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "labels_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "creator": { + "$ref": "user.json" + }, + "open_issues": { + "type": ["null", "integer"] + }, + "closed_issues": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "due_on": { + "type": ["null", "string"], + "format": "date-time" + } + } }, "active_lock_reason": { "type": ["null", "string"] @@ -90,25 +167,64 @@ "merge_commit_sha": { "type": ["null", "string"] }, - "assignee_id": { - "type": ["null", "integer"] + "assignee": { + "$ref": "user.json" }, "assignees": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "$ref": "user.json" } }, "requested_reviewers": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "$ref": "user.json" } }, "requested_teams": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "slug": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "privacy": { + "type": ["null", "string"] + }, + "permission": { + "type": ["null", "string"] + }, + "members_url": { + "type": ["null", "string"] + }, + "repositories_url": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "object"], + "properties": {} + } + } } }, "head": { diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json b/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json index e61814804e4f..6c290db20f8d 100644 --- a/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/releases.json @@ -55,8 +55,8 @@ "type": ["null", "string"], "format": "date-time" }, - "author_id": { - "type": ["null", "integer"] + "author": { + "$ref": "user.json" }, "assets": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index b092489f97c8..70e6870e97f9 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -24,7 +24,7 @@ import re -from typing import Any, List, Mapping, Tuple +from typing import Any, Dict, List, Mapping, Tuple from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode @@ -88,6 +88,43 @@ def _get_authenticator(token: str): tokens = [t.strip() for t in token.split(TOKEN_SEPARATOR)] return MultipleTokenAuthenticator(tokens=tokens, auth_method="token") + @staticmethod + def _get_branches_data(selected_branches: str, full_refresh_args: Dict[str, Any] = None) -> Tuple[Dict[str, str], Dict[str, List[str]]]: + selected_branches = set(filter(None, selected_branches.split(" "))) + + # Get the default branch for each repository + default_branches = {} + repository_stats_stream = RepositoryStats(**full_refresh_args) + for stream_slice in repository_stats_stream.stream_slices(sync_mode=SyncMode.full_refresh): + default_branches.update( + { + repo_stats["full_name"]: repo_stats["default_branch"] + for repo_stats in repository_stats_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + } + ) + + all_branches = [] + branches_stream = Branches(**full_refresh_args) + for stream_slice in branches_stream.stream_slices(sync_mode=SyncMode.full_refresh): + for branch in branches_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): + all_branches.append(f"{branch['repository']}/{branch['name']}") + + # Create mapping of repository to list of branches to pull commits for + # If no branches are specified for a repo, use its default branch + branches_to_pull: Dict[str, List[str]] = {} + for repo in full_refresh_args["repositories"]: + repo_branches = [] + for branch in selected_branches: + branch_parts = branch.split("/", 2) + if "/".join(branch_parts[:2]) == repo and branch in all_branches: + repo_branches.append(branch_parts[-1]) + if not repo_branches: + repo_branches = [default_branches[repo]] + + branches_to_pull[repo] = repo_branches + + return default_branches, branches_to_pull + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: authenticator = self._get_authenticator(config["access_token"]) @@ -110,6 +147,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: full_refresh_args = {"authenticator": authenticator, "repositories": repositories} incremental_args = {**full_refresh_args, "start_date": config["start_date"]} organization_args = {"authenticator": authenticator, "organizations": organizations} + default_branches, branches_to_pull = self._get_branches_data(config.get("branch", ""), full_refresh_args) return [ Assignees(**full_refresh_args), @@ -118,7 +156,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Comments(**incremental_args), CommitCommentReactions(**incremental_args), CommitComments(**incremental_args), - Commits(**incremental_args), + Commits(**incremental_args, branches_to_pull=branches_to_pull, default_branches=default_branches), Events(**incremental_args), IssueCommentReactions(**incremental_args), IssueEvents(**incremental_args), diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index 35ccbcb4e641..166e52a162bd 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -23,6 +23,11 @@ "description": "The date from which you'd like to replicate data for GitHub in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only in the following incremental streams: comments, commits and issues.", "examples": ["2021-03-01T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "branch": { + "type": "string", + "examples": ["airbytehq/airbyte/master"], + "description": "Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled." } } } diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index 0bf9c41f0fec..a6a5d076433b 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -56,6 +56,16 @@ class GithubStream(HttpStream, ABC): cache = request_cache() url_base = "https://api.github.com/" + # To prevent dangerous behavior, the `vcr` library prohibits the use of nested caching. + # Here's an example of dangerous behavior: + # cache = Cassette.use('whatever') + # with cache: + # with cache: + # pass + # + # Therefore, we will only use `cache` for the top-level stream, so as not to cause possible difficulties. + top_level_stream = True + primary_key = "id" # GitHub pagination could be from 1 to 100. @@ -63,11 +73,6 @@ class GithubStream(HttpStream, ABC): stream_base_params = {} - # Fields in below variable will be used for data clearing. Put there keys which represent: - # - objects `{}`, like `user`, `actor` etc. - # - lists `[]`, like `labels`, `assignees` etc. - fields_to_minimize = () - def __init__(self, repositories: List[str], **kwargs): super().__init__(**kwargs) self.repositories = repositories @@ -110,7 +115,11 @@ def backoff_time(self, response: requests.Response) -> Union[int, float]: def read_records(self, stream_slice: Mapping[str, any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: try: - yield from super().read_records(stream_slice=stream_slice, **kwargs) + if self.top_level_stream: + with self.cache: + yield from super().read_records(stream_slice=stream_slice, **kwargs) + else: + yield from super().read_records(stream_slice=stream_slice, **kwargs) except HTTPError as e: error_msg = str(e) @@ -128,8 +137,28 @@ def read_records(self, stream_slice: Mapping[str, any] = None, **kwargs) -> Iter ) elif e.response.status_code == requests.codes.NOT_FOUND and "/teams?" in error_msg: # For private repositories `Teams` stream is not available and we get "404 Client Error: Not Found for - # url: https://api.github.com/orgs/sherifnada/teams?per_page=100" error. - error_msg = f"Syncing `Team` stream isn't available for repository `{stream_slice['repository']}`." + # url: https://api.github.com/orgs//teams?per_page=100" error. + error_msg = f"Syncing `Team` stream isn't available for organization `{stream_slice['organization']}`." + elif e.response.status_code == requests.codes.NOT_FOUND and "/repos?" in error_msg: + # `Repositories` stream is not available for repositories not in an organization. + # Handle "404 Client Error: Not Found for url: https://api.github.com/orgs//repos?per_page=100" error. + error_msg = f"Syncing `Repositories` stream isn't available for organization `{stream_slice['organization']}`." + elif e.response.status_code == requests.codes.GONE and "/projects?" in error_msg: + # Some repos don't have projects enabled and we we get "410 Client Error: Gone for + # url: https://api.github.com/repos/xyz/projects?per_page=100" error. + error_msg = f"Syncing `Projects` stream isn't available for repository `{stream_slice['repository']}`." + elif e.response.status_code == requests.codes.NOT_FOUND and "/orgs/" in error_msg: + # Some streams are not available for repositories owned by a user instead of an organization. + # Handle "404 Client Error: Not Found" errors + if isinstance(self, Repositories): + error_msg = f"Syncing `Repositories` stream isn't available for organization `{stream_slice['organization']}`." + elif isinstance(self, Users): + error_msg = f"Syncing `Users` stream isn't available for organization `{stream_slice['organization']}`." + elif isinstance(self, Organizations): + error_msg = f"Syncing `Organizations` stream isn't available for organization `{stream_slice['organization']}`." + else: + self.logger.error(f"Undefined error while reading records: {error_msg}") + raise e elif e.response.status_code == requests.codes.CONFLICT: error_msg = ( f"Syncing `{self.name}` stream isn't available for repository " @@ -171,34 +200,6 @@ def parse_response( yield self.transform(record=record, repository=stream_slice["repository"]) def transform(self, record: MutableMapping[str, Any], repository: str = None, organization: str = None) -> MutableMapping[str, Any]: - """ - Use this method to: - - remove excessive fields from record; - - minify subelements in the record. For example, if you have `reviews` record which looks like this: - { - "id": 671782869, - "node_id": "MDE3OlB1bGxSZXF1ZXN0UmV2aWV3NjcxNzgyODY5", - "user": { - "login": "keu", - "id": 1619536, - ... - }, - "body": "lgtm, just small comment", - ... - } - - `user` subelement contains almost all possible fields fo user and it's not optimal to store such data in - `reviews` record. We may leave only `user.id` field and save in to `user_id` field in the record. So if you - need to do something similar with your record you may use this method. - """ - for field in self.fields_to_minimize: - field_value = record.pop(field, None) - if field_value is None: - record[field] = field_value - elif isinstance(field_value, dict): - record[f"{field}_id"] = field_value.get("id") if field_value else None - elif isinstance(field_value, list): - record[field] = [value.get("id") for value in field_value] if repository: record["repository"] = repository if organization: @@ -310,6 +311,8 @@ class PullRequestStats(GithubStream): API docs: https://docs.github.com/en/rest/reference/pulls#get-a-pull-request """ + top_level_stream = False + @property def record_keys(self) -> List[str]: return list(self.get_json_schema()["properties"].keys()) @@ -338,6 +341,8 @@ class Reviews(GithubStream): API docs: https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request """ + top_level_stream = False + def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: @@ -455,7 +460,6 @@ class Releases(SemiIncrementalGithubStream): """ cursor_field = "created_at" - fields_to_minimize = ("author",) def transform(self, record: MutableMapping[str, Any], repository: str = None, **kwargs) -> MutableMapping[str, Any]: record = super().transform(record=record, repository=repository) @@ -474,11 +478,6 @@ class Events(SemiIncrementalGithubStream): """ cursor_field = "created_at" - fields_to_minimize = ( - "actor", - "repo", - "org", - ) class PullRequests(SemiIncrementalGithubStream): @@ -487,14 +486,6 @@ class PullRequests(SemiIncrementalGithubStream): """ page_size = 50 - fields_to_minimize = ( - "milestone", - "assignee", - "labels", - "assignees", - "requested_reviewers", - "requested_teams", - ) def __init__(self, **kwargs): super().__init__(**kwargs) @@ -505,8 +496,7 @@ def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iter Decide if this a first read or not by the presence of the state object """ self._first_read = not bool(stream_state) - with self.cache: - yield from super().read_records(stream_state=stream_state, **kwargs) + yield from super().read_records(stream_state=stream_state, **kwargs) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/pulls" @@ -551,7 +541,6 @@ class IssueMilestones(SemiIncrementalGithubStream): """ is_sorted_descending = True - fields_to_minimize = ("creator",) stream_base_params = { "state": "all", "sort": "updated", @@ -593,7 +582,6 @@ class Projects(SemiIncrementalGithubStream): API docs: https://docs.github.com/en/rest/reference/projects#list-repository-projects """ - fields_to_minimize = ("creator",) stream_base_params = { "state": "all", } @@ -613,10 +601,6 @@ class IssueEvents(SemiIncrementalGithubStream): """ cursor_field = "created_at" - fields_to_minimize = ( - "actor", - "issue", - ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/issues/events" @@ -638,27 +622,105 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Commits(IncrementalGithubStream): """ - API docs: https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository + API docs: https://docs.github.com/en/rest/reference/repos#list-commits + + Pull commits from each branch of each repository, tracking state for each branch """ primary_key = "sha" cursor_field = "created_at" - fields_to_minimize = ( - "author", - "committer", - ) - def transform(self, record: MutableMapping[str, Any], repository: str = None, **kwargs) -> MutableMapping[str, Any]: + def __init__(self, branches_to_pull: Mapping[str, List[str]], default_branches: Mapping[str, str], **kwargs): + super().__init__(**kwargs) + self.branches_to_pull = branches_to_pull + self.default_branches = default_branches + + def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super(IncrementalGithubStream, self).request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) + params["since"] = self.get_starting_point( + stream_state=stream_state, repository=stream_slice["repository"], branch=stream_slice["branch"] + ) + params["sha"] = stream_slice["branch"] + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for stream_slice in super().stream_slices(**kwargs): + repository = stream_slice["repository"] + for branch in self.branches_to_pull.get(repository, []): + yield {"branch": branch, "repository": repository} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + for record in response.json(): # GitHub puts records in an array. + yield self.transform(record=record, repository=stream_slice["repository"], branch=stream_slice["branch"]) + + def transform(self, record: MutableMapping[str, Any], repository: str = None, branch: str = None, **kwargs) -> MutableMapping[str, Any]: record = super().transform(record=record, repository=repository) # Record of the `commits` stream doesn't have an updated_at/created_at field at the top level (so we could # just write `record["updated_at"]` or `record["created_at"]`). Instead each record has such value in # `commit.author.date`. So the easiest way is to just enrich the record returned from API with top level # field `created_at` and use it as cursor_field. + # Include the branch in the record record["created_at"] = record["commit"]["author"]["date"] + record["branch"] = branch return record + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + state_value = latest_cursor_value = latest_record.get(self.cursor_field) + current_repository = latest_record["repository"] + current_branch = latest_record["branch"] + + if current_stream_state.get(current_repository): + repository_commits_state = current_stream_state[current_repository] + if repository_commits_state.get(self.cursor_field): + # transfer state from old source version to per-branch version + if current_branch == self.default_branches[current_repository]: + state_value = max(latest_cursor_value, repository_commits_state[self.cursor_field]) + del repository_commits_state[self.cursor_field] + elif repository_commits_state.get(current_branch, {}).get(self.cursor_field): + state_value = max(latest_cursor_value, repository_commits_state[current_branch][self.cursor_field]) + + if current_repository not in current_stream_state: + current_stream_state[current_repository] = {} + + current_stream_state[current_repository][current_branch] = {self.cursor_field: state_value} + return current_stream_state + + def get_starting_point(self, stream_state: Mapping[str, Any], repository: str, branch: str) -> str: + start_point = self._start_date + if stream_state and stream_state.get(repository, {}).get(branch, {}).get(self.cursor_field): + return max(start_point, stream_state[repository][branch][self.cursor_field]) + if branch == self.default_branches[repository]: + return super().get_starting_point(stream_state=stream_state, repository=repository) + return start_point + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + repository = stream_slice["repository"] + start_point_map = { + branch: self.get_starting_point(stream_state=stream_state, repository=repository, branch=branch) + for branch in self.branches_to_pull.get(repository, []) + } + for record in super(SemiIncrementalGithubStream, self).read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ): + if record.get(self.cursor_field) > start_point_map[stream_slice["branch"]]: + yield record + elif self.is_sorted_descending and record.get(self.cursor_field) < start_point_map[stream_slice["branch"]]: + break + class Issues(IncrementalGithubStream): """ @@ -667,12 +729,6 @@ class Issues(IncrementalGithubStream): page_size = 50 # `issues` is a large stream so it's better to set smaller page size. - fields_to_minimize = ( - "assignee", - "milestone", - "labels", - "assignees", - ) stream_base_params = { "state": "all", "sort": "updated", @@ -697,6 +753,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class ReactionStream(GithubStream, ABC): parent_key = "id" + top_level_stream = False def __init__(self, **kwargs): self._stream_kwargs = deepcopy(kwargs) diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index aaf783fdaa31..699f08441558 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index 08322d5aa37b..8a59d1a3a9ef 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -101,6 +101,9 @@ def get_json_schema(self) -> Dict[str, Any]: else: output_type = [google_datatype_mapping.get(google_data_type, "string"), "null"] field_value = {"type": output_type} + if google_data_type == "DATE": + field_value["format"] = "date" + local_json_schema["properties"][field] = field_value return local_json_schema diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_performance_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_performance_report.json index 6187ad22def2..5b653a2c0493 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_performance_report.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_performance_report.json @@ -100,7 +100,7 @@ }, "segments.date": { "type": ["null", "string"], - "format": "datetime" + "format": "date" }, "segments.day_of_week": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json index c38ef7a2ce48..2dc46c51c474 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json @@ -139,7 +139,7 @@ }, "segments.date": { "type": ["null", "string"], - "format": "datetime" + "format": "date" }, "segments.day_of_week": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json index 3a9aff288bfe..75ece36c594a 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json @@ -136,7 +136,7 @@ }, "segments.date": { "type": ["null", "string"], - "format": "datetime" + "format": "date" }, "segments.day_of_week": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json index bcf02ad89fc8..4e92ede6d75c 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json @@ -4,7 +4,7 @@ "properties": { "segments.date": { "type": ["null", "string"], - "format": "datetime" + "format": "date" }, "segments.day_of_week": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index 00d2bcc8e1a6..217496b57004 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json index 2f133c335ac6..93c0ce5c25a4 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { - "credentials_json": "", + "credentials": { "credentials_json": "" }, "view_id": "211669975", "start_date": "2021-02-11", "window_in_days": 1, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_config.json index 8b28cda3c7cf..9134e6fbf53c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/integration_tests/sample_config.json @@ -1,5 +1,4 @@ { - "credentials_json": "credentials_json", "view_id": "id", "start_date": "start_date", "window_in_days": 1, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index aa4f19e11286..a1994a5ffbfd 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -404,28 +404,54 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late class GoogleAnalyticsOauth2Authenticator(Oauth2Authenticator): - """Request example for API token extraction: + """ + This class supports either default authorization_code and JWT OAuth + authorizations in case of service account. + + Request example for API token extraction: curl --location --request POST https://oauth2.googleapis.com/token?grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion=signed_JWT """ + use_jwt_auth: bool = False + def __init__(self, config): - self.credentials_json = json.loads(config["credentials_json"]) - self.client_email = self.credentials_json["client_email"] + client_secret, client_id, refresh_token = None, None, None + if "credentials_json" in config: + # Backward compatability with previous config format. Use + # credentials_json from config root. + auth = config + else: + auth = config["credentials"] + if "credentials_json" in auth: + # Service account JWT authorization + self.use_jwt_auth = True + credentials_json = json.loads(auth["credentials_json"]) + client_secret, client_id, refresh_token = credentials_json["private_key"], credentials_json["private_key_id"], None + self.client_email = credentials_json["client_email"] + else: + # OAuth 2.0 authorization_code authorization + client_secret, client_id, refresh_token = auth["client_secret"], auth["client_id"], auth["refresh_token"] self.scope = "https://www.googleapis.com/auth/analytics.readonly" super().__init__( token_refresh_endpoint="https://oauth2.googleapis.com/token", - client_secret=self.credentials_json["private_key"], - client_id=self.credentials_json["private_key_id"], - refresh_token=None, + client_secret=client_secret, + client_id=client_id, + refresh_token=refresh_token, + scopes=[self.scope], ) def refresh_access_token(self) -> Tuple[str, int]: """ - Calling the Google OAuth 2.0 token endpoint. Used for authorizing signed JWT. - Returns tuple with access token and token's time-to-live + Calling the Google OAuth 2.0 token endpoint. Used for authorizing + with signed JWT if credentials_json provided by config. Otherwise use + default OAuth2.0 workflow. + :return tuple with access token and token's time-to-live. """ + if not self.use_jwt_auth: + return super().refresh_access_token() + response_json = None try: response = requests.request(method="POST", url=self.token_refresh_endpoint, params=self.get_refresh_request_params()) @@ -446,6 +472,7 @@ def refresh_access_token(self) -> Tuple[str, int]: def get_refresh_request_params(self) -> Mapping[str, any]: """ Sign the JWT with RSA-256 using the private key found in service account JSON file. + Not used with default OAuth2.0 authorization_code grant_type. """ token_lifetime = 3600 # token lifetime is 1 hour @@ -460,12 +487,9 @@ def get_refresh_request_params(self) -> Mapping[str, any]: "iat": issued_at, "exp": expiration_time, } - headers = {"kid": self.client_id} - signed_jwt = jwt.encode(payload, self.client_secret, headers=headers, algorithm="RS256") - - return {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": str(signed_jwt)} + return {"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": signed_jwt} class SourceGoogleAnalyticsV4(AbstractSource): diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index dca528311c14..7231817271a2 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -4,14 +4,48 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google Analytics V4 Spec", "type": "object", - "required": ["credentials_json", "view_id", "start_date"], - "additionalProperties": false, + "required": ["credentials", "view_id", "start_date"], + "additionalProperties": true, "properties": { - "credentials_json": { - "type": "string", - "title": "Credentials JSON", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", - "airbyte_secret": true + "credentials": { + "title": "Authentication mechanism", + "type": "object", + "description": "Choose either OAuth2.0 flow or provide your own JWT credentials for service account", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0 authorization", + "properties": { + "option_title": { + "type": "string", + "const": "Default OAuth2.0 authorization" + }, + "client_id": { "type": "string" }, + "client_secret": { "type": "string", "airbyte_secret": true }, + "refresh_token": { "type": "string", "airbyte_secret": true } + }, + "required": ["client_id", "client_secret", "refresh_token"], + "additionalProperties": false + }, + { + "type": "object", + "title": "Service Account Key", + "properties": { + "option_title": { + "type": "string", + "const": "Service account credentials" + }, + "credentials_json": { + "type": "string", + "title": "Credentials JSON", + "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key.", + "airbyte_secret": true + } + }, + "required": ["credentials_json"], + "additionalProperties": true + } + ] }, "view_id": { "type": "string", @@ -37,5 +71,15 @@ "description": "A JSON array describing the custom reports you want to sync from GA. Check out the docs to get more information about this field." } } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "oauthFlowInitParameters": [ + ["credentials", "client_id"], + ["credentials", "client_secret"], + ["credentials", "refresh_token"] + ] + } } } diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index a338df5d25c5..d20861d32588 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -25,6 +25,7 @@ import json from pathlib import Path from unittest.mock import MagicMock, patch +from urllib.parse import unquote import pytest from airbyte_cdk.sources.streams.http.auth import NoAuth @@ -47,6 +48,11 @@ def mock_metrics_dimensions_type_list_link(requests_mock): ) +@pytest.fixture +def mock_auth_call(requests_mock): + yield requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": "", "expires_in": 0}) + + def test_metrics_dimensions_type_list(mock_metrics_dimensions_type_list_link): test_metrics, test_dimensions = GoogleAnalyticsV4TypesList().read_records(sync_mode=None) @@ -79,22 +85,31 @@ def test_lookup_metrics_dimensions_data_type(metrics_dimensions_mapping, mock_me assert test == expected -class GoogleAnalyticsOauth2AuthenticatorMock: - def refresh_access_token(self): - return MagicMock(), 0 +@patch("source_google_analytics_v4.source.jwt") +def test_check_connection_jwt(jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call): + test_config = json.loads(read_file("../integration_tests/sample_config.json")) + del test_config["custom_reports"] + test_config["credentials"] = {"credentials_json": '{"client_email": "", "private_key": "", "private_key_id": ""}'} + source = SourceGoogleAnalyticsV4() + assert source.check_connection(MagicMock(), test_config) == (True, None) + jwt_encode_mock.encode.assert_called() + assert mock_auth_call.called -@patch( - "source_google_analytics_v4.source.GoogleAnalyticsOauth2Authenticator.refresh_access_token", - new=GoogleAnalyticsOauth2AuthenticatorMock.refresh_access_token, -) -def test_check_connection(mocker, mock_metrics_dimensions_type_list_link): +@patch("source_google_analytics_v4.source.jwt") +def test_check_connection_oauth(jwt_encode_mock, mocker, mock_metrics_dimensions_type_list_link, mock_auth_call): test_config = json.loads(read_file("../integration_tests/sample_config.json")) - test_config["credentials_json"] = '{"client_email": "", "private_key": "", "private_key_id": ""}' - del test_config["custom_reports"] - + test_config["credentials"] = { + "client_id": "client_id_val", + "client_secret": "client_secret_val", + "refresh_token": "refresh_token_val", + } source = SourceGoogleAnalyticsV4() - logger_mock, config_mock = MagicMock(), test_config - - assert source.check_connection(logger_mock, config_mock) == (True, None) + assert source.check_connection(MagicMock(), test_config) == (True, None) + jwt_encode_mock.encode.assert_not_called() + assert "https://www.googleapis.com/auth/analytics.readonly" in unquote(mock_auth_call.last_request.body) + assert "client_id_val" in unquote(mock_auth_call.last_request.body) + assert "client_secret_val" in unquote(mock_auth_call.last_request.body) + assert "refresh_token_val" in unquote(mock_auth_call.last_request.body) + assert mock_auth_call.called diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile index 82af90a630eb..f37cd64e047a 100755 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ b/airbyte-integrations/connectors/source-google-search-console/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index db4e6e0cf321..c600c1690562 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -46,9 +46,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> try: stream_kwargs = self.get_stream_kwargs(config) sites = Sites(**stream_kwargs) - stream_slice = next(sites.stream_slices(SyncMode.full_refresh)) - sites_gen = sites.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) - next(sites_gen) + stream_slice = sites.stream_slices(SyncMode.full_refresh) + + # stream_slice returns all site_urls and we need to make sure that + # the connection is successful for all of them + for _slice in stream_slice: + sites_gen = sites.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice) + next(sites_gen) return True, None except Exception as error: diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json index e060196070b4..4717c7c57598 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json @@ -28,6 +28,7 @@ "type": "object", "oneOf": [ { + "title": "Client Auth", "type": "object", "required": [ "auth_type", @@ -38,7 +39,10 @@ "properties": { "auth_type": { "type": "string", - "const": "Client" + "const": "Client", + "enum": ["Client"], + "default": "Client", + "order": 0 }, "client_id": { "type": "string", @@ -59,11 +63,15 @@ }, { "type": "object", + "title": "Service Auth", "required": ["auth_type", "service_account_info"], "properties": { "auth_type": { "type": "string", - "const": "Service" + "const": "Service", + "enum": ["Service"], + "default": "Service", + "order": 0 }, "service_account_info": { "type": "string", diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile index 7d479263db13..5a6a2776284e 100644 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-greenhouse/Dockerfile @@ -14,5 +14,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml new file mode 100644 index 000000000000..e8054a566055 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-greenhouse:dev +tests: + spec: + - spec_path: "source_greenhouse/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_users_only.json" + status: "succeed" + - config_path: "integration_tests/config_invalid.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + - config_path: "secrets/config_users_only.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_users_only.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config_users_only.json" + configured_catalog_path: "integration_tests/configured_catalog_users_only.json" diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_client.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py similarity index 60% rename from airbyte-integrations/connectors/source-greenhouse/unit_tests/test_client.py rename to airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py index d7949296e0f7..d6cbdc97c495 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/acceptance.py @@ -24,25 +24,11 @@ import pytest -from grnhse.exceptions import EndpointNotFound, HTTPError -from source_greenhouse.client import Client +pytest_plugins = ("source_acceptance_test.plugin",) -def test__heal_check_with_wrong_api_key(): - client = Client(api_key="wrong_key") - alive, error = client.health_check() - assert not alive - assert error == '401 {"message":"Invalid Basic Auth credentials"}' - - -def test__custom_fields_with_wrong_api_key(): - client = Client(api_key="wrong_key") - with pytest.raises(HTTPError, match='401 {"message":"Invalid Basic Auth credentials"}'): - list(client.list("custom_fields")) - - -def test_client_wrong_endpoint(): - client = Client(api_key="wrong_key") - with pytest.raises(EndpointNotFound, match="unknown_endpoint"): - next(client.list("unknown_endpoint")) +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/config_invalid.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/config_invalid.json new file mode 100644 index 000000000000..73c1f28072e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/config_invalid.json @@ -0,0 +1,3 @@ +{ + "api_key": "bla" +} diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..67e9ea328282 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json @@ -0,0 +1,114 @@ +{ + "streams": [ + { + "stream": { + "name": "applications", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "candidates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "close_reasons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "degrees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "departments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "job_posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "offers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "scorecards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json new file mode 100644 index 000000000000..6a3c2bc4b197 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-greenhouse/requirements.txt b/airbyte-integrations/connectors/source-greenhouse/requirements.txt index dd447512e620..e74f41a28ce1 100644 --- a/airbyte-integrations/connectors/source-greenhouse/requirements.txt +++ b/airbyte-integrations/connectors/source-greenhouse/requirements.txt @@ -1,4 +1,5 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python +-e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index 5fb46438a36e..8bef623cc40f 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -25,12 +25,19 @@ from setuptools import find_packages, setup +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] setup( name="source_greenhouse", description="Source implementation for Greenhouse.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "six==1.15.0", "grnhse-api==0.1.1", "pytest==6.1.2"], + install_requires=["airbyte-protocol", "base-python", "six==1.15.0", "grnhse-api==0.1.1"], package_data={"": ["*.json", "schemas/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, ) diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/client.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/client.py index 43b441288567..ae0dda1c8db9 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/client.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/client.py @@ -24,9 +24,10 @@ from functools import partial -from typing import Mapping, Tuple +from typing import Generator, List, Mapping, Tuple -from base_python import BaseClient +from airbyte_protocol import AirbyteStream +from base_python import AirbyteLogger, BaseClient from grnhse import Harvest from grnhse.exceptions import HTTPError @@ -67,16 +68,40 @@ def list(self, name, **kwargs): def _enumerate_methods(self) -> Mapping[str, callable]: return {entity: partial(self.list, name=entity) for entity in self.ENTITIES} + def get_accessible_endpoints(self) -> List[str]: + """Try to read each supported endpoint and return accessible stream names""" + logger = AirbyteLogger() + accessible_endpoints = [] + for entity in self.ENTITIES: + try: + getattr(self._client, entity).get() + accessible_endpoints.append(entity) + except HTTPError as error: + logger.warn(f"Endpoint '{entity}' error: {str(error)}") + if "This API Key does not have permission for this endpoint" not in str(error): + raise error + logger.info(f"API key has access to {len(accessible_endpoints)} endpoints: {accessible_endpoints}") + return accessible_endpoints + def health_check(self) -> Tuple[bool, str]: alive = True error_msg = None - try: - # because there is no good candidate to try our connection - # we use users endpoint as potentially smallest dataset - self._client.users.get() + accessible_endpoints = self.get_accessible_endpoints() + if not accessible_endpoints: + alive = False + error_msg = "Your API Key does not have permission for any existing endpoints. Please grant read permissions for required streams/endpoints" + except HTTPError as error: alive = False error_msg = str(error) return alive, error_msg + + @property + def streams(self) -> Generator[AirbyteStream, None, None]: + """Process accessible streams only""" + accessible_endpoints = self.get_accessible_endpoints() + for stream in super().streams: + if stream.name in accessible_endpoints: + yield stream diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/applications.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/applications.json index a47166854c3f..b31409c5d9ea 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/applications.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/applications.json @@ -41,7 +41,7 @@ "type": ["null", "string"] }, "prospect_owner": { - "type": "object", + "type": ["null", "object"], "properties": { "name": { "type": "string" @@ -69,7 +69,15 @@ "type": "integer" }, "current_stage": { - "type": ["null", "string"] + "type": ["null", "object"], + "properties": { + "name": { + "type": "string" + }, + "id": { + "type": "integer" + } + } }, "credited_to": { "type": "object", diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json index 55b57c0816f4..cb7149d3a7ed 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json @@ -109,7 +109,7 @@ "type": ["null", "string"] }, "prospect_owner": { - "type": "object", + "type": ["null", "object"], "properties": { "name": { "type": "string" @@ -137,7 +137,15 @@ "type": "integer" }, "current_stage": { - "type": ["null", "string"] + "type": ["null", "object"], + "properties": { + "name": { + "type": "string" + }, + "id": { + "type": "integer" + } + } }, "credited_to": { "type": "object", diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json index f8c6f6609dd4..de168a63abbc 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/users.json @@ -3,43 +3,43 @@ "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "first_name": { - "type": "string" + "type": ["null", "string"] }, "last_name": { - "type": "string" + "type": ["null", "string"] }, "primary_email_address": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "disabled": { - "type": "boolean" + "type": ["null", "boolean"] }, "site_admin": { - "type": "boolean" + "type": ["null", "boolean"] }, "emails": { - "type": "array", + "type": ["null", "array"], "items": { - "type": "string" + "type": ["null", "string"] } }, "employee_id": { "type": ["null", "integer"] }, "linked_candidate_ids": { - "type": "array" + "type": ["null", "array"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index d69f512961fd..2edd9c054d8c 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -14,5 +14,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh" -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=0.1.15 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh old mode 100644 new mode 100755 index e4d8b1cef896..7ad3352709bd --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py index e1da609e4816..12b14d535124 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py @@ -58,7 +58,7 @@ def __init__(self, start_date, credentials, **kwargs): "contact_lists": ContactListStream(**common_params), "contacts": CRMObjectStream(entity="contact", **common_params), "deal_pipelines": DealPipelineStream(**common_params), - "deals": DealStream(**common_params), + "deals": DealStream(associations=["contacts"], **common_params), "email_events": EmailEventStream(**common_params), "engagements": EngagementStream(**common_params), "forms": FormStream(**common_params), diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json index ac5549e4f007..5187593975f5 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { + "id": { + "type": "string" + }, "vid": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json index 301ce47fe1de..f6564e9bb731 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json @@ -327,7 +327,7 @@ "contacts": { "type": ["null", "array"], "items": { - "type": ["null", "integer"] + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json index be7e57cea5e2..15150ec585d3 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json @@ -2,13 +2,10 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "ownerId": { - "type": ["null", "integer"] + "id": { + "type": ["null", "string"] }, - "type": { + "email": { "type": ["null", "string"] }, "firstName": { @@ -17,52 +14,33 @@ "lastName": { "type": ["null", "string"] }, - "email": { - "type": ["null", "string"] + "userId": { + "type": ["null", "integer"] }, "createdAt": { - "type": ["null", "string"] - }, - "signature": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "updatedAt": { - "type": ["null", "string"] - }, - "hasContactsAccess": { - "type": ["null", "boolean"] + "type": ["null", "string"], + "format": "date-time" }, - "isActive": { + "archived": { "type": ["null", "boolean"] }, - "activeUserId": { - "type": ["null", "integer"] - }, - "userIdIncludingInactive": { - "type": ["null", "integer"] - }, - "remoteList": { + "teams": { "type": ["null", "array"], "items": { - "type": ["null", "object"], + "type": "object", "properties": { "id": { - "type": ["null", "integer"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "ownerId": { - "type": ["null", "integer"] - }, - "remoteId": { "type": ["null", "string"] }, - "remoteType": { + "name": { "type": ["null", "string"] }, - "active": { - "type": ["null", "boolean"] + "membership": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile index 0a60fd900fe7..50dd9d320d14 100644 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -34,5 +34,5 @@ COPY source_intercom ./source_intercom ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json old mode 100644 new mode 100755 index 1bb4ec8dd2e0..8259cd09e6ff --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -1,20 +1,20 @@ { "companies": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "company_segments": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "conversations": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "conversation_parts": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "contacts": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 }, "segments": { - "updated_at": "2022-07-12T10:44:09+00:00" + "updated_at": 7626086649 } } diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json old mode 100644 new mode 100755 index e3e8395eaea0..e5de1448e423 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -1,20 +1,20 @@ { "companies": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "company_segments": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "conversations": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "conversation_parts": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "contacts": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 }, "segments": { - "updated_at": "2019-07-12T11:22:46+00:00" + "updated_at": 1626086649 } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json old mode 100644 new mode 100755 index 0f72dc70babf..5b07257d2951 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json @@ -21,8 +21,7 @@ "type": ["null", "integer"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "monthly_spend": { "type": ["null", "number"], diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json index e5a6d4cc8229..a9b902f1521b 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json @@ -55,8 +55,7 @@ "type": ["null", "boolean"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json old mode 100644 new mode 100755 index 67f1f02387ca..ca08be8a7923 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json @@ -21,8 +21,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json old mode 100644 new mode 100755 index 38944193a20c..f343927b3661 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -72,8 +72,7 @@ "type": ["null", "integer"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "signed_up_at": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json old mode 100644 new mode 100755 index 8b5ad38dc3f5..10621c781d2f --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json @@ -113,8 +113,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "redacted": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json old mode 100644 new mode 100755 index 68ae977e9ceb..2be8a0eb19f4 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json @@ -416,8 +416,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] }, "user": { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json old mode 100644 new mode 100755 index 67f1f02387ca..ca08be8a7923 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json @@ -21,8 +21,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py old mode 100644 new mode 100755 index 479a6da50873..ebae32aaa8de --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -126,13 +126,6 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, record = super().parse_response(response, stream_state, **kwargs) for record in record: - updated_at = record.get(self.cursor_field) - - if updated_at: - record[self.cursor_field] = datetime.fromtimestamp( - record[self.cursor_field] - ).isoformat() # convert timestamp to datetime string - yield from self.filter_by_state(stream_state=stream_state, record=record) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: @@ -335,6 +328,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") + config["start_date"] = datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%SZ").timestamp() + auth = TokenAuthenticator(token=config["access_token"]) return [ Admins(authenticator=auth, **config), diff --git a/airbyte-integrations/connectors/source-kafka/Dockerfile b/airbyte-integrations/connectors/source-kafka/Dockerfile new file mode 100644 index 000000000000..47adceed70bc --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/Dockerfile @@ -0,0 +1,12 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-kafka + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.name=airbyte/source-kafka +LABEL io.airbyte.version=0.1.0 diff --git a/airbyte-integrations/connectors/source-kafka/README.md b/airbyte-integrations/connectors/source-kafka/README.md new file mode 100644 index 000000000000..ba2d9ad6ea27 --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/README.md @@ -0,0 +1,45 @@ +# Kafka Source + +This is the repository for the Kafka source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/kafka). + +## Local development + +### Prerequisites +* If you are using Python for connector development, minimal required version `= 3.7.0` +* Valid credentials (see the "Create credentials section for instructions) +TODO: _which languages and tools does a user need to develop on this connector? add them to the bullet list above_ + +### Iteration +TODO: _which commands should a developer use to run this connector locally?_ + +### Testing +#### Unit Tests +TODO: _how can a user run unit tests?_ + +#### Integration Tests +TODO: _how can a user run integration tests?_ +_this section is currently under construction -- please reach out to us on Slack for help with setting up Airbyte's standard test suite_ + + +### Locally running the connector docker image + +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/kafka:dev +``` + +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-kafka:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kafka:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kafka:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-kafka:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/kafka) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `spec.json` file. `secrets` is gitignored by default. + +**If you are an Airbyte core member**, copy the credentials from Lastpass under the secret name `source kafka test creds` +and place them into `secrets/config.json`. diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle new file mode 100644 index 000000000000..5fcb74099344 --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -0,0 +1,24 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.source.kafka.KafkaSource' +} + +dependencies { + implementation project(':airbyte-config:models') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:bases:base-java') + + implementation 'org.apache.kafka:kafka-clients:2.8.0' + implementation 'org.apache.kafka:connect-json:2.8.0' + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-kafka') + integrationTestJavaImplementation "org.testcontainers:kafka:1.15.3" + + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaProtocol.java b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaProtocol.java new file mode 100644 index 000000000000..32dcdeb1b387 --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaProtocol.java @@ -0,0 +1,33 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.kafka; + +public enum KafkaProtocol { + + PLAINTEXT, + SASL_PLAINTEXT, + SASL_SSL + +} diff --git a/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java new file mode 100644 index 000000000000..cdb4879282e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java @@ -0,0 +1,145 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.kafka; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Lists; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.BaseConnector; +import io.airbyte.integrations.base.*; +import io.airbyte.protocol.models.*; +import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KafkaSource extends BaseConnector implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(KafkaSource.class); + + public KafkaSource() {} + + @Override + public AirbyteConnectionStatus check(JsonNode config) { + try { + final String testTopic = config.has("test_topic") ? config.get("test_topic").asText() : ""; + if (!testTopic.isBlank()) { + final KafkaSourceConfig kafkaSourceConfig = KafkaSourceConfig.getKafkaSourceConfig(config); + final KafkaConsumer consumer = kafkaSourceConfig.getCheckConsumer(); + consumer.subscribe(Pattern.compile(testTopic)); + consumer.listTopics(); + consumer.close(); + LOGGER.info("Successfully connected to Kafka brokers for topic '{}'.", config.get("test_topic").asText()); + } + return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); + } catch (Exception e) { + LOGGER.error("Exception attempting to connect to the Kafka brokers: ", e); + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage("Could not connect to the Kafka brokers with provided configuration. \n" + e.getMessage()); + } + } + + @Override + public AirbyteCatalog discover(JsonNode config) throws Exception { + + Set topicsToSubscribe = KafkaSourceConfig.getKafkaSourceConfig(config).getTopicsToSubscribe(); + List streams = topicsToSubscribe.stream().map(topic -> CatalogHelpers + .createAirbyteStream(topic, Field.of("value", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))) + .collect(Collectors.toList()); + return new AirbyteCatalog().withStreams(streams); + } + + @Override + public AutoCloseableIterator read(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) throws Exception { + final AirbyteConnectionStatus check = check(config); + if (check.getStatus().equals(AirbyteConnectionStatus.Status.FAILED)) { + throw new RuntimeException("Unable establish a connection: " + check.getMessage()); + } + + final KafkaSourceConfig kafkaSourceConfig = KafkaSourceConfig.getKafkaSourceConfig(config); + final KafkaConsumer consumer = kafkaSourceConfig.getConsumer(); + List> recordsList = new ArrayList<>(); + + int retry = config.has("repeated_calls") ? config.get("repeated_calls").intValue() : 0; + int pollCount = 0; + while (true) { + final ConsumerRecords consumerRecords = consumer.poll(Duration.of(100, ChronoUnit.MILLIS)); + if (consumerRecords.count() == 0) { + pollCount++; + if (pollCount > retry) { + break; + } + } + + consumerRecords.forEach(record -> { + LOGGER.info("Consumer Record: key - {}, value - {}, partition - {}, offset - {}", + record.key(), record.value(), record.partition(), record.offset()); + recordsList.add(record); + }); + consumer.commitAsync(); + } + consumer.close(); + Iterator> iterator = recordsList.iterator(); + + return AutoCloseableIterators.fromIterator(new AbstractIterator<>() { + + @Override + protected AirbyteMessage computeNext() { + if (iterator.hasNext()) { + ConsumerRecord record = iterator.next(); + return new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(record.topic()) + .withEmittedAt(Instant.now().toEpochMilli()) + .withData(record.value())); + } + + return endOfData(); + } + + }); + } + + public static void main(String[] args) throws Exception { + final Source source = new KafkaSource(); + LOGGER.info("Starting source: {}", KafkaSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("Completed source: {}", KafkaSource.class); + } + +} diff --git a/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSourceConfig.java b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSourceConfig.java new file mode 100644 index 000000000000..7343a44d5803 --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSourceConfig.java @@ -0,0 +1,159 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.kafka; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.apache.kafka.clients.CommonClientConfigs; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.TopicPartition; +import org.apache.kafka.common.config.SaslConfigs; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.connect.json.JsonDeserializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class KafkaSourceConfig { + + protected static final Logger LOGGER = LoggerFactory.getLogger(KafkaSourceConfig.class); + private static KafkaSourceConfig instance; + private final JsonNode config; + private KafkaConsumer consumer; + private Set topicsToSubscribe; + + private KafkaSourceConfig(JsonNode config) { + this.config = config; + } + + public static KafkaSourceConfig getKafkaSourceConfig(JsonNode config) { + if (instance == null) { + instance = new KafkaSourceConfig(config); + } + return instance; + } + + private KafkaConsumer buildKafkaConsumer(JsonNode config) { + final Map props = new HashMap<>(); + props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, config.get("bootstrap_servers").asText()); + props.put(ConsumerConfig.GROUP_ID_CONFIG, + config.has("group_id") ? config.get("group_id").asText() : null); + props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, + config.has("max_poll_records") ? config.get("max_poll_records").intValue() : null); + props.putAll(propertiesByProtocol(config)); + props.put(ConsumerConfig.CLIENT_ID_CONFIG, + config.has("client_id") ? config.get("client_id").asText() : null); + props.put(ConsumerConfig.CLIENT_DNS_LOOKUP_CONFIG, config.get("client_dns_lookup").asText()); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, config.get("enable_auto_commit").booleanValue()); + props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, + config.has("auto_commit_interval_ms") ? config.get("auto_commit_interval_ms").intValue() : null); + props.put(ConsumerConfig.RETRY_BACKOFF_MS_CONFIG, + config.has("retry_backoff_ms") ? config.get("retry_backoff_ms").intValue() : null); + props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, + config.has("request_timeout_ms") ? config.get("request_timeout_ms").intValue() : null); + props.put(ConsumerConfig.RECEIVE_BUFFER_CONFIG, + config.has("receive_buffer_bytes") ? config.get("receive_buffer_bytes").intValue() : null); + props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName()); + props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class.getName()); + + final Map filteredProps = props.entrySet().stream() + .filter(entry -> entry.getValue() != null && !entry.getValue().toString().isBlank()) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + return new KafkaConsumer<>(filteredProps); + } + + private Map propertiesByProtocol(JsonNode config) { + JsonNode protocolConfig = config.get("protocol"); + LOGGER.info("Kafka protocol config: {}", protocolConfig.toString()); + final KafkaProtocol protocol = KafkaProtocol.valueOf(protocolConfig.get("security_protocol").asText().toUpperCase()); + final ImmutableMap.Builder builder = ImmutableMap.builder() + .put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, protocol.toString()); + + switch (protocol) { + case PLAINTEXT -> {} + case SASL_SSL, SASL_PLAINTEXT -> { + builder.put(SaslConfigs.SASL_JAAS_CONFIG, config.get("sasl_jaas_config").asText()); + builder.put(SaslConfigs.SASL_MECHANISM, config.get("sasl_mechanism").asText()); + } + default -> throw new RuntimeException("Unexpected Kafka protocol: " + Jsons.serialize(protocol)); + } + + return builder.build(); + } + + public KafkaConsumer getConsumer() { + if (consumer != null) { + return consumer; + } + consumer = buildKafkaConsumer(config); + + JsonNode subscription = config.get("subscription"); + LOGGER.info("Kafka subscribe method: {}", subscription.toString()); + switch (subscription.get("subscription_type").asText()) { + case "subscribe" -> { + String topicPattern = subscription.get("topic_pattern").asText(); + consumer.subscribe(Pattern.compile(topicPattern)); + topicsToSubscribe = consumer.listTopics().keySet().stream() + .filter(topic -> topic.matches(topicPattern)) + .collect(Collectors.toSet()); + } + case "assign" -> { + topicsToSubscribe = new HashSet<>(); + String topicPartitions = subscription.get("topic_partitions").asText(); + String[] topicPartitionsStr = topicPartitions.replaceAll("\\s+", "").split(","); + List topicPartitionList = Arrays.stream(topicPartitionsStr).map(topicPartition -> { + String[] pair = topicPartition.split(":"); + topicsToSubscribe.add(pair[0]); + return new TopicPartition(pair[0], Integer.parseInt(pair[1])); + }).collect(Collectors.toList()); + LOGGER.info("Topic-partition list: {}", topicPartitionList); + consumer.assign(topicPartitionList); + } + } + return consumer; + } + + public Set getTopicsToSubscribe() { + if (topicsToSubscribe == null) { + getConsumer(); + } + return topicsToSubscribe; + } + + public KafkaConsumer getCheckConsumer() { + return buildKafkaConsumer(config); + } + +} diff --git a/airbyte-integrations/connectors/source-kafka/src/main/resources/spec.json b/airbyte-integrations/connectors/source-kafka/src/main/resources/spec.json new file mode 100644 index 000000000000..d88e807c243e --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/src/main/resources/spec.json @@ -0,0 +1,212 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/kafka", + "supportsIncremental": true, + "supportsNormalization": false, + "supportsDBT": false, + "supported_source_sync_modes": ["append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Kafka Source Spec", + "type": "object", + "required": ["bootstrap_servers", "subscription", "protocol"], + "additionalProperties": false, + "properties": { + "bootstrap_servers": { + "title": "Bootstrap servers", + "description": "A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. The client will make use of all servers irrespective of which servers are specified here for bootstrapping—this list only impacts the initial hosts used to discover the full set of servers. This list should be in the form host1:port1,host2:port2,.... Since these servers are just used for the initial connection to discover the full cluster membership (which may change dynamically), this list need not contain the full set of servers (you may want more than one, though, in case a server is down).", + "type": "string", + "examples": ["kafka-broker1:9092,kafka-broker2:9092"] + }, + "subscription": { + "title": "Subscribe method", + "type": "object", + "description": "You can choose to manually assign a list of partitions, or subscribe to all topics matching specified pattern to get dynamically assigned partitions", + "oneOf": [ + { + "title": "Manually assign a list of partitions", + "required": ["subscription_type", "topic_partitions"], + "properties": { + "subscription_type": { + "description": "Manually assign a list of partitions to this consumer. This interface does not allow for incremental assignment and will replace the previous assignment (if there is one).\nIf the given list of topic partitions is empty, it is treated the same as unsubscribe().", + "type": "string", + "const": "assign", + "enum": ["assign"], + "default": "assign" + }, + "topic_partitions": { + "title": "List of topic:partition pairs", + "type": "string", + "examples": ["sample.topic:0, sample.topic:1"] + } + } + }, + { + "title": "Subscribe to all topics matching specified pattern", + "required": ["subscription_type", "topic_pattern"], + "properties": { + "subscription_type": { + "description": "Topic pattern from which the records will be read.", + "type": "string", + "const": "subscribe", + "enum": ["subscribe"], + "default": "subscribe" + }, + "topic_pattern": { + "title": "Topic pattern", + "type": "string", + "examples": ["sample.topic"] + } + } + } + ] + }, + "test_topic": { + "title": "Test topic", + "description": "Topic to test if Airbyte can consume messages.", + "type": "string", + "examples": ["test.topic"] + }, + "group_id": { + "title": "Group ID", + "description": "Group id.", + "type": "string", + "examples": ["group.id"] + }, + "max_poll_records": { + "title": "Max poll records", + "description": "The maximum number of records returned in a single call to poll(). Note, that max_poll_records does not impact the underlying fetching behavior. The consumer will cache the records from each fetch request and returns them incrementally from each poll.", + "type": "integer", + "default": 500 + }, + "protocol": { + "title": "Protocol", + "type": "object", + "description": "Protocol used to communicate with brokers.", + "oneOf": [ + { + "title": "PLAINTEXT", + "required": ["security_protocol"], + "properties": { + "security_protocol": { + "type": "string", + "enum": ["PLAINTEXT"], + "default": "PLAINTEXT" + } + } + }, + { + "title": "SASL PLAINTEXT", + "required": [ + "security_protocol", + "sasl_mechanism", + "sasl_jaas_config" + ], + "properties": { + "security_protocol": { + "type": "string", + "enum": ["SASL_PLAINTEXT"], + "default": "SASL_PLAINTEXT" + }, + "sasl_mechanism": { + "title": "SASL mechanism", + "description": "SASL mechanism used for client connections. This may be any mechanism for which a security provider is available.", + "type": "string", + "default": "PLAIN", + "enum": ["PLAIN"] + }, + "sasl_jaas_config": { + "title": "SASL JAAS config", + "description": "JAAS login context parameters for SASL connections in the format used by JAAS configuration files.", + "type": "string", + "default": "", + "airbyte_secret": true + } + } + }, + { + "title": "SASL SSL", + "required": [ + "security_protocol", + "sasl_mechanism", + "sasl_jaas_config" + ], + "properties": { + "security_protocol": { + "type": "string", + "enum": ["SASL_SSL"], + "default": "SASL_SSL" + }, + "sasl_mechanism": { + "title": "SASL mechanism", + "description": "SASL mechanism used for client connections. This may be any mechanism for which a security provider is available.", + "type": "string", + "default": "GSSAPI", + "enum": ["GSSAPI", "OAUTHBEARER", "SCRAM-SHA-256"] + }, + "sasl_jaas_config": { + "title": "SASL JAAS config", + "description": "JAAS login context parameters for SASL connections in the format used by JAAS configuration files.", + "type": "string", + "default": "", + "airbyte_secret": true + } + } + } + ] + }, + "client_id": { + "title": "Client ID", + "description": "An id string to pass to the server when making requests. The purpose of this is to be able to track the source of requests beyond just ip/port by allowing a logical application name to be included in server-side request logging.", + "type": "string", + "examples": ["airbyte-consumer"] + }, + "enable_auto_commit": { + "title": "Enable auto commit", + "description": "If true the consumer's offset will be periodically committed in the background.", + "type": "boolean", + "default": true + }, + "auto_commit_interval_ms": { + "title": "Auto commit interval ms", + "description": "The frequency in milliseconds that the consumer offsets are auto-committed to Kafka if enable.auto.commit is set to true.", + "type": "integer", + "default": 5000 + }, + "client_dns_lookup": { + "title": "Client DNS lookup", + "description": "Controls how the client uses DNS lookups. If set to use_all_dns_ips, connect to each returned IP address in sequence until a successful connection is established. After a disconnection, the next IP is used. Once all IPs have been used once, the client resolves the IP(s) from the hostname again. If set to resolve_canonical_bootstrap_servers_only, resolve each bootstrap address into a list of canonical names. After the bootstrap phase, this behaves the same as use_all_dns_ips. If set to default (deprecated), attempt to connect to the first IP address returned by the lookup, even if the lookup returns multiple IP addresses.", + "type": "string", + "default": "use_all_dns_ips", + "enum": [ + "default", + "use_all_dns_ips", + "resolve_canonical_bootstrap_servers_only" + ] + }, + "retry_backoff_ms": { + "title": "Retry backoff ms", + "description": "The amount of time to wait before attempting to retry a failed request to a given topic partition. This avoids repeatedly sending requests in a tight loop under some failure scenarios.", + "type": "integer", + "default": 100 + }, + "request_timeout_ms": { + "title": "Request timeout ms", + "description": "The configuration controls the maximum amount of time the client will wait for the response of a request. If the response is not received before the timeout elapses the client will resend the request if necessary or fail the request if retries are exhausted.", + "type": "integer", + "default": 30000 + }, + "receive_buffer_bytes": { + "title": "Receive buffer bytes", + "description": "The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. If the value is -1, the OS default will be used.", + "type": "integer", + "default": 32768 + }, + "repeated_calls": { + "title": "Repeated calls", + "description": "The number of repeated calls to poll() if no messages were received.", + "type": "integer", + "default": 3 + } + } + } +} diff --git a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java new file mode 100644 index 000000000000..f4b23ec6b743 --- /dev/null +++ b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java @@ -0,0 +1,100 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.kafka; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.jackson.MoreMappers; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.*; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import org.testcontainers.containers.KafkaContainer; +import org.testcontainers.utility.DockerImageName; + +public class KafkaSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final ObjectMapper mapper = MoreMappers.initMapper(); + private static final String TOPIC_NAME = "test.topic"; + + private static KafkaContainer KAFKA; + + @Override + protected String getImageName() { + return "airbyte/source-kafka:dev"; + } + + @Override + protected JsonNode getConfig() { + ObjectNode stubProtocolConfig = mapper.createObjectNode(); + stubProtocolConfig.put("security_protocol", KafkaProtocol.PLAINTEXT.toString()); + + return Jsons.jsonNode(ImmutableMap.builder() + .put("bootstrap_servers", KAFKA.getBootstrapServers()) + .put("topic_pattern", TOPIC_NAME) + .build()); + } + + @Override + protected void setupEnvironment(TestDestinationEnv environment) throws Exception { + KAFKA = new KafkaContainer(DockerImageName.parse("confluentinc/cp-kafka:6.2.0")); + KAFKA.start(); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + KAFKA.close(); + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { + return CatalogHelpers.createConfiguredAirbyteCatalog( + TOPIC_NAME, + null, + Field.of("value", JsonSchemaPrimitive.STRING)); + } + + @Override + protected JsonNode getState() throws Exception { + return Jsons.jsonNode(new HashMap<>()); + } + + @Override + protected List getRegexTests() throws Exception { + return Collections.emptyList(); + } + +} diff --git a/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-config.yml b/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-config.yml index cdc599a71f8c..b1be4556f38d 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-lever-hiring:dev tests: spec: - - spec_path: "source_lever_hiring/spec.json" + - spec_path: "integration_tests/spec.json" connection: - config_path: "secrets/config.json" status: "succeed" @@ -15,13 +15,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.txt" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/abnormal_state.json index 52b0f2c2118f..63bc51dffcaf 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" + "opportunities": { + "updatedAt": 1638587931515000 } } diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/configured_catalog.json index 0dff92bd05a6..a2c5d92fb92f 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/configured_catalog.json @@ -1,3 +1,77 @@ { - "streams": [] + "streams": [ + { + "stream": { + "name": "applications", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "interviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "notes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "offers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "opportunities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "referrals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] } diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/invalid_config.json index f3732995784f..589ce69812f7 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/invalid_config.json @@ -1,3 +1,7 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" + "client_id": "fake_client_id", + "client_secret": "fake_client_secret", + "refresh_token": "fake_refresh_token", + "environment": "Sandbox", + "start_date": "2021-07-12T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json index ecc4913b84c7..2730c7b0ee81 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_config.json @@ -1,3 +1,7 @@ { - "fix-me": "TODO" + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + "environment": "Sandbox", + "start_date": "2021-07-12T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_state.json index 3587e579822d..c87e935590ee 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "value" + "opportunities": { + "updatedAt": 1628587931515 } } diff --git a/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json new file mode 100644 index 000000000000..89aa5932e3da --- /dev/null +++ b/airbyte-integrations/connectors/source-lever-hiring/integration_tests/spec.json @@ -0,0 +1,51 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/lever-hiring", + "changelogUrl": "https://docs.airbyte.io/integrations/sources/lever-hiring#changelog", + "connectionSpecification": { + "title": "Lever Hiring Spec", + "type": "object", + "properties": { + "client_id": { + "title": "Client Id", + "description": "The client application id as provided when registering the application with Lever.", + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "The application secret as provided when registering the application with Lever.", + "airbyte_secret": true, + "type": "string" + }, + "refresh_token": { + "title": "Refresh Token", + "description": "The refresh token your application will need to submit to get a new access token after it's expired.", + "type": "string" + }, + "environment": { + "title": "Environment", + "description": "Sandbox or Production environment.", + "default": "Production", + "enum": ["Sandbox", "Production"], + "type": "string" + }, + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2019-02-25T00:00:00Z. Any data before this date will not be replicated.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2021-04-25T00:00:00Z"], + "type": "string" + } + }, + "required": ["client_id", "client_secret", "refresh_token", "start_date"] + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "oauthFlowInitParameters": [ + ["client_id"], + ["client_secret"], + ["refresh_token"] + ] + } + } +} diff --git a/airbyte-integrations/connectors/source-lever-hiring/setup.py b/airbyte-integrations/connectors/source-lever-hiring/setup.py index 488db43a8444..672d08305115 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/setup.py +++ b/airbyte-integrations/connectors/source-lever-hiring/setup.py @@ -33,6 +33,7 @@ "pytest~=6.1", "pytest-mock~=3.6.1", "source-acceptance-test", + "responses~=0.13.3", ] setup( @@ -42,7 +43,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/schemas.py b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/schemas.py new file mode 100644 index 000000000000..afb253d6a6f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/schemas.py @@ -0,0 +1,193 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from typing import Any, List, MutableMapping, Optional, Type + +import pydantic +from pydantic import BaseModel +from pydantic.typing import resolve_annotations + + +class AllOptional(pydantic.main.ModelMetaclass): + def __new__(self, name, bases, namespaces, **kwargs): + """ + Iterate through fields and wrap then with typing.Optional type. + """ + annotations = resolve_annotations(namespaces.get("__annotations__", {}), namespaces.get("__module__", None)) + for base in bases: + annotations = {**annotations, **getattr(base, "__annotations__", {})} + for field in annotations: + if not field.startswith("__"): + annotations[field] = Optional[annotations[field]] + namespaces["__annotations__"] = annotations + return super().__new__(self, name, bases, namespaces, **kwargs) + + +class BaseSchemaModel(BaseModel, metaclass=AllOptional): + class Config: + arbitrary_types_allowed = True + + @classmethod + def schema_extra(cls, schema: MutableMapping[str, Any], model: Type["BaseModel"]) -> None: + # Remove auto generated "title" and "description" fields, because they do not carry absolutely any payload. + schema.pop("title", None) + schema.pop("description", None) + # Remove required section so any missing attribute from API wont break object validation. + schema.pop("required", None) + for name, prop in schema.get("properties", {}).items(): + prop.pop("title", None) + prop.pop("description", None) + allow_none = model.__fields__[name].allow_none + if allow_none: + if "type" in prop: + prop["type"] = ["null", prop["type"]] + elif "$ref" in prop: + ref = prop.pop("$ref") + prop["oneOf"] = [{"type": "null"}, {"$ref": ref}] + + +class Application(BaseSchemaModel): + id: str + type: str + candidateId: str + opportunityId: str + posting: str + postingHiringManager: str + postingOwner: str + name: str + company: str + phone: dict + email: str + links: List[str] + comments: str + user: str + customQuestions: List[dict] + createdAt: int + archived: dict + requisitionForHire: dict + + +class Interview(BaseSchemaModel): + id: str + panel: str + subject: str + note: str + interviewers: List[dict] + timezone: str + createdAt: int + date: int + duration: int + location: str + feedbackTemplate: str + feedbackForms: List[str] + feedbackReminder: str + user: str + stage: str + canceledAt: int + postings: List[str] + gcalEventUrl: str + + +class Note(BaseSchemaModel): + id: str + text: str + fields: List[dict] + user: str + secret: bool + completedAt: int + deletedAt: int + createdAt: int + + +class Offer(BaseSchemaModel): + id: str + posting: str + createdAt: int + status: str + creator: str + fields: List[dict] + signatures: dict + approved: str + approvedAt: int + sentAt: int + sentDocument: dict + signedDocument: dict + + +class Opportunity(BaseSchemaModel): + id: str + name: str + contact: str + headline: str + stage: str + confidentiality: str + location: str + phones: List[dict] + emails: List[str] + links: List[str] + archived: dict + tags: List[str] + sources: List[str] + stageChanges: List[dict] + origin: str + sourcedBy: str + owner: str + followers: List[str] + applications: List[str] + createdAt: int + updatedAt: int + lastInteractionAt: int + lastAdvancedAt: int + snoozedUntil: int + urls: dict + resume: str + dataProtection: dict + isAnonymized: bool + + +class Referral(BaseSchemaModel): + id: str + type: str + text: str + instructions: str + fields: List[dict] + baseTemplateId: str + user: str + referrer: str + stage: str + createdAt: int + completedAt: int + + +class User(BaseSchemaModel): + id: str + name: str + username: str + email: str + accessRole: str + photo: str + createdAt: int + deactivatedAt: int + externalDirectoryId: str + linkedContactIds: List[str] diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py index 0a531fd0379f..b4ed9a724996 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/source.py @@ -25,13 +25,80 @@ from typing import Any, List, Mapping, Tuple +from airbyte_cdk.models import AuthSpecification, ConnectorSpecification, OAuth2Specification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from pydantic import Field +from pydantic.main import BaseModel + +from .streams import Applications, Interviews, Notes, Offers, Opportunities, Referrals, Users + + +class ConnectorConfig(BaseModel): + class Config: + title = "Lever Hiring Spec" + + client_id: str = Field( + description="The client application id as provided when registering the application with Lever.", + ) + client_secret: str = Field( + description="The application secret as provided when registering the application with Lever.", + airbyte_secret=True, + ) + refresh_token: str = Field( + description="The refresh token your application will need to submit to get a new access token after it's expired.", + ) + environment: str = Field(description="Sandbox or Production environment.", enum=["Sandbox", "Production"], default="Production") + start_date: str = Field( + description="UTC date and time in the format 2019-02-25T00:00:00Z. Any data before this date will not be replicated.", + pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + examples=["2021-04-25T00:00:00Z"], + ) class SourceLeverHiring(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - pass + URL_MAP_ACCORDING_ENVIRONMENT = { + "Sandbox": {"login": "https://sandbox-lever.auth0.com/", "api": "https://api.sandbox.lever.co/"}, + "Production": {"login": "https://auth.lever.co/", "api": "https://api.lever.co/"}, + } + + def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, any]: + authenticator = Oauth2Authenticator( + token_refresh_endpoint=f"{self.URL_MAP_ACCORDING_ENVIRONMENT[config['environment']]['login']}oauth/token", + client_id=config["client_id"], + client_secret=config["client_secret"], + refresh_token=config["refresh_token"], + ) + _ = authenticator.get_auth_header() + return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - pass + authenticator = Oauth2Authenticator( + token_refresh_endpoint=f"{self.URL_MAP_ACCORDING_ENVIRONMENT[config['environment']]['login']}oauth/token", + client_id=config["client_id"], + client_secret=config["client_secret"], + refresh_token=config["refresh_token"], + ) + full_refresh_params = {"authenticator": authenticator, "base_url": self.URL_MAP_ACCORDING_ENVIRONMENT[config["environment"]]["api"]} + stream_params_with_start_date = {**full_refresh_params, "start_date": config["start_date"]} + return [ + Applications(**stream_params_with_start_date), + Interviews(**stream_params_with_start_date), + Notes(**stream_params_with_start_date), + Offers(**stream_params_with_start_date), + Opportunities(**stream_params_with_start_date), + Referrals(**stream_params_with_start_date), + Users(**full_refresh_params), + ] + + def spec(self, *args, **kwargs) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.io/integrations/sources/lever-hiring", + changelogUrl="https://docs.airbyte.io/integrations/sources/lever-hiring#changelog", + connectionSpecification=ConnectorConfig.schema(), + authSpecification=AuthSpecification( + auth_type="oauth2.0", + oauth2Specification=OAuth2Specification(oauthFlowInitParameters=[["client_id"], ["client_secret"], ["refresh_token"]]), + ), + ) diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json deleted file mode 100644 index 6cec44844b80..000000000000 --- a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/spec.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Lever Hiring Spec", - "type": "object", - "required": ["TODO"], - "additionalProperties": false, - "properties": { - "TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.": { - "type": "string", - "description": "describe me" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/streams.py b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/streams.py index 717ae65c8b05..ac47e9b0ab81 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/streams.py +++ b/airbyte-integrations/connectors/source-lever-hiring/source_lever_hiring/streams.py @@ -22,74 +22,151 @@ # SOFTWARE. # -from abc import ABC +from abc import ABC, abstractmethod from typing import Any, Iterable, Mapping, MutableMapping, Optional +import pendulum import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from .schemas import Application, BaseSchemaModel, Interview, Note, Offer, Opportunity, Referral, User + class LeverHiringStream(HttpStream, ABC): - url_base = "" + primary_key = "id" + page_size = 50 + + stream_params = {} + API_VERSION = "v1" + + def __init__(self, base_url: str, **kwargs): + super().__init__(**kwargs) + self.base_url = base_url + + @property + def url_base(self) -> str: + return f"{self.base_url}/{self.API_VERSION}/" + + def path(self, **kwargs) -> str: + return self.name + + @property + @abstractmethod + def schema(self) -> BaseSchemaModel: + """Pydantic model that represents stream schema""" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} + response_data = response.json() + if response_data.get("hasNext"): + return {"offset": response_data["next"]} + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"limit": self.page_size} + params.update(self.stream_params) + if next_page_token: + params.update(next_page_token) + return params def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - yield {} + yield from response.json()["data"] + + def get_json_schema(self) -> Mapping[str, Any]: + """Use Pydantic schema""" + return self.schema.schema() class IncrementalLeverHiringStream(LeverHiringStream, ABC): + + state_checkpoint_interval = 100 + cursor_field = "updatedAt" + + def __init__(self, start_date: str, **kwargs): + super().__init__(**kwargs) + self._start_ts = int(pendulum.parse(start_date).timestamp()) * 1000 + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + state_ts = int(current_stream_state.get(self.cursor_field, 0)) + return {self.cursor_field: max(latest_record.get(self.cursor_field), state_ts)} + + def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs): + stream_state = stream_state or {} + params = super().request_params(stream_state=stream_state, **kwargs) + state_ts = int(stream_state.get(self.cursor_field, 0)) + params["updated_at_start"] = max(state_ts, self._start_ts) + + return params + + +class Opportunities(IncrementalLeverHiringStream): """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. + Opportunities stream: https://hire.lever.co/developer/documentation#list-all-opportunities """ - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None + schema = Opportunity + base_params = {"include": "followers", "confidentiality": "all"} - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - :return str: The name of the cursor field. - """ - return "" +class Users(LeverHiringStream): + """ + Users stream: https://hire.lever.co/developer/documentation#list-all-users + """ - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} + schema = User + base_params = {"includeDeactivated": True} + + +class OpportynityChildStream(LeverHiringStream, ABC): + def __init__(self, start_date: str, **kwargs): + super().__init__(**kwargs) + self._start_date = start_date + + def path(self, stream_slice: Mapping[str, any] = None, **kwargs) -> str: + return f"opportunities/{stream_slice['opportunity_id']}/{self.name}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for stream_slice in super().stream_slices(**kwargs): + opportunities_stream = Opportunities(authenticator=self.authenticator, base_url=self.base_url, start_date=self._start_date) + for opportunity in opportunities_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): + yield {"opportunity_id": opportunity["id"]} + + +class Applications(OpportynityChildStream): + """ + Applications stream: https://hire.lever.co/developer/documentation#list-all-applications + """ + + schema = Application + + +class Interviews(OpportynityChildStream): + """ + Interviews stream: https://hire.lever.co/developer/documentation#list-all-interviews + """ + + schema = Interview + + +class Notes(OpportynityChildStream): + """ + Notes stream: https://hire.lever.co/developer/documentation#list-all-notes + """ + + schema = Note + + +class Offers(OpportynityChildStream): + """ + Offers stream: https://hire.lever.co/developer/documentation#list-all-offers + """ + + schema = Offer + + +class Referrals(OpportynityChildStream): + """ + Referrals stream: https://hire.lever.co/developer/documentation#list-all-referrals + """ + + schema = Referral diff --git a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py new file mode 100644 index 000000000000..35ff1105f42e --- /dev/null +++ b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/conftest.py @@ -0,0 +1,79 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from pytest import fixture + + +@fixture +def test_config(): + return { + "client_id": "test_client_id", + "client_secret": "test_client_secret", + "refresh_token": "test_refresh_token", + "environment": "Sandbox", + "start_date": "2021-05-07T00:00:00Z", + } + + +@fixture +def test_full_refresh_config(): + return {"base_url": "test_base_url"} + + +@fixture +def test_incremental_config(): + return {"base_url": "test_base_url", "start_date": "2020-01-01T00:00:00Z"} + + +@fixture +def test_opportunity_record(): + return { + "id": "test_id", + "name": "test_name", + "contact": "test_contact", + "headline": "test_headline", + "stage": "test_stage", + "confidentiality": "non-confidential", + "location": "test_location", + "phones": [{"type": "test_mobile", "value": "test_value"}], + "emails": ["test_emails"], + "links": ["test_link_1", "test_link_2"], + "archived": {"reason": "test_reason", "archivedAt": 1628513942512}, + "tags": [], + "sources": ["test_source_1"], + "stageChanges": [{"toStageId": "test_lead-new", "toStageIndex": 0, "updatedAt": 1628509001183, "userId": "test_userId"}], + "origin": "test_origin", + "sourcedBy": "test_sourcedBy", + "owner": "test_owner", + "followers": ["test_follower"], + "applications": ["test_application"], + "createdAt": 1738509001183, + "updatedAt": 1738542849132, + "lastInteractionAt": 1738513942512, + "lastAdvancedAt": 1738513942512, + "snoozedUntil": None, + "urls": {"list": "https://hire.sandbox.lever.co/candidates", "show": "https://hire.sandbox.lever.co/candidates/test_show"}, + "isAnonymized": False, + "dataProtection": None, + } diff --git a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_incremental_streams.py index de66bb4a3f04..b016089176b4 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_incremental_streams.py @@ -22,14 +22,12 @@ # SOFTWARE. # -# from unittest.mock import MagicMock - +import pytest from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_lever_hiring.source import IncrementalLeverHiringStream +from source_lever_hiring.streams import IncrementalLeverHiringStream -@fixture +@pytest.fixture def patch_incremental_base_class(mocker): # Mock abstract methods to enable instantiating abstract class mocker.patch.object(IncrementalLeverHiringStream, "path", "v0/example_endpoint") @@ -37,46 +35,39 @@ def patch_incremental_base_class(mocker): mocker.patch.object(IncrementalLeverHiringStream, "__abstractmethods__", set()) -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalLeverHiringStream() +def test_cursor_field(patch_incremental_base_class, test_incremental_config): + stream = IncrementalLeverHiringStream(**test_incremental_config) # TODO: replace this with your expected cursor field - expected_cursor_field = [] + expected_cursor_field = "updatedAt" assert stream.cursor_field == expected_cursor_field -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalLeverHiringStream() - # expected_cursor_field = [] - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} +def test_get_updated_state(patch_incremental_base_class, test_incremental_config, test_opportunity_record): + stream = IncrementalLeverHiringStream(**test_incremental_config) + inputs = {"current_stream_state": {"opportunities": {"updatedAt": 1600000000000}}, "latest_record": test_opportunity_record} + expected_state = {"updatedAt": 1738542849132} assert stream.get_updated_state(**inputs) == expected_state -def test_stream_slices(patch_incremental_base_class): - stream = IncrementalLeverHiringStream() - # expected_cursor_field = [] - # TODO: replace this with your input parameters - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list +def test_stream_slices(patch_incremental_base_class, test_incremental_config): + stream = IncrementalLeverHiringStream(**test_incremental_config) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": ["updatedAt"], "stream_state": {"updatedAt": 1600000000000}} expected_stream_slice = [None] assert stream.stream_slices(**inputs) == expected_stream_slice -def test_supports_incremental(patch_incremental_base_class, mocker): +def test_supports_incremental(patch_incremental_base_class, mocker, test_incremental_config): mocker.patch.object(IncrementalLeverHiringStream, "cursor_field", "dummy_field") - stream = IncrementalLeverHiringStream() + stream = IncrementalLeverHiringStream(**test_incremental_config) assert stream.supports_incremental -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalLeverHiringStream() +def test_source_defined_cursor(patch_incremental_base_class, test_incremental_config): + stream = IncrementalLeverHiringStream(**test_incremental_config) assert stream.source_defined_cursor -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalLeverHiringStream() - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None +def test_stream_checkpoint_interval(patch_incremental_base_class, test_incremental_config): + stream = IncrementalLeverHiringStream(**test_incremental_config) + expected_checkpoint_interval = 100 assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py index 8f49f36a20c4..a174120a7231 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_source.py @@ -24,19 +24,30 @@ from unittest.mock import MagicMock +import responses from source_lever_hiring.source import SourceLeverHiring -def test_check_connection(mocker): +def setup_responses(): + responses.add( + responses.POST, + "https://sandbox-lever.auth0.com/oauth/token", + json={"access_token": "fake_access_token", "expires_in": 3600}, + ) + + +@responses.activate +def test_check_connection(test_config): + setup_responses() source = SourceLeverHiring() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) + logger_mock = MagicMock() + assert source.check_connection(logger_mock, test_config) == (True, None) -def test_streams(mocker): +@responses.activate +def test_streams(test_config): + setup_responses() source = SourceLeverHiring() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 + streams = source.streams(test_config) + expected_streams_number = 7 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_streams.py index de2e35185059..e8d408ffaeac 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-lever-hiring/unit_tests/test_streams.py @@ -26,7 +26,48 @@ from unittest.mock import MagicMock import pytest -from source_lever_hiring.source import LeverHiringStream +import requests +import responses +from source_lever_hiring.streams import LeverHiringStream + + +def setup_responses(): + responses.add( + responses.GET, + "https://api.sandbox.lever.co/v0/example_endpoint", + json={ + "data": [ + { + "id": "fake_id", + "name": "fake_name", + "contact": "fake_contact", + "headline": "Airbyte", + "stage": "offer", + "confidentiality": "non-confidential", + "location": "Los Angeles, CA", + "origin": "referred", + "createdAt": 1628510997134, + "updatedAt": 1628542848755, + "isAnonymized": False, + }, + { + "id": "fake_id_2", + "name": "fake_name_2", + "contact": "fake_contact_2", + "headline": "Airbyte", + "stage": "applicant-new", + "confidentiality": "non-confidential", + "location": "Los Angeles, CA", + "origin": "sourced", + "createdAt": 1628509001183, + "updatedAt": 1628542849132, + "isAnonymized": False, + }, + ], + "hasNext": True, + "next": "%5B1628543173558%2C%227bf8c1ac-4a68-450f-bea0-a1e2c3f5aeaf%22%5D", + }, + ) @pytest.fixture @@ -37,45 +78,59 @@ def patch_base_class(mocker): mocker.patch.object(LeverHiringStream, "__abstractmethods__", set()) -def test_request_params(patch_base_class): - stream = LeverHiringStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} +def test_request_params(patch_base_class, test_full_refresh_config): + stream = LeverHiringStream(**test_full_refresh_config) + inputs = { + "stream_slice": {"slice": "test_slice"}, + "stream_state": {"updatedAt": 1600000000000}, + "next_page_token": {"offset": "next_page_cursor"}, + } + expected_params = {"limit": stream.page_size, "offset": "next_page_cursor"} assert stream.request_params(**inputs) == expected_params -def test_next_page_token(patch_base_class): - stream = LeverHiringStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None +@responses.activate +def test_next_page_token(patch_base_class, test_full_refresh_config): + setup_responses() + stream = LeverHiringStream(**test_full_refresh_config) + inputs = {"response": requests.get("https://api.sandbox.lever.co/v0/example_endpoint")} + expected_token = {"offset": "%5B1628543173558%2C%227bf8c1ac-4a68-450f-bea0-a1e2c3f5aeaf%22%5D"} assert stream.next_page_token(**inputs) == expected_token -def test_parse_response(patch_base_class): - stream = LeverHiringStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} +@responses.activate +def test_parse_response(patch_base_class, test_full_refresh_config): + setup_responses() + stream = LeverHiringStream(**test_full_refresh_config) + inputs = {"response": requests.get("https://api.sandbox.lever.co/v0/example_endpoint")} + expected_parsed_object = { + "id": "fake_id", + "name": "fake_name", + "contact": "fake_contact", + "headline": "Airbyte", + "stage": "offer", + "confidentiality": "non-confidential", + "location": "Los Angeles, CA", + "origin": "referred", + "createdAt": 1628510997134, + "updatedAt": 1628542848755, + "isAnonymized": False, + } assert next(stream.parse_response(**inputs)) == expected_parsed_object -def test_request_headers(patch_base_class): - stream = LeverHiringStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - # expected_headers = {} +def test_request_headers(patch_base_class, test_full_refresh_config): + stream = LeverHiringStream(**test_full_refresh_config) + inputs = { + "stream_slice": {"slice": "test_slice"}, + "stream_state": {"updatedAt": 1600000000000}, + "next_page_token": {"offset": "next_page_cursor"}, + } assert stream.request_headers(**inputs) == {} -def test_http_method(patch_base_class): - stream = LeverHiringStream() - # TODO: replace this with your expected http request method +def test_http_method(patch_base_class, test_full_refresh_config): + stream = LeverHiringStream(**test_full_refresh_config) expected_method = "GET" assert stream.http_method == expected_method @@ -89,15 +144,15 @@ def test_http_method(patch_base_class): (HTTPStatus.INTERNAL_SERVER_ERROR, True), ], ) -def test_should_retry(patch_base_class, http_status, should_retry): +def test_should_retry(patch_base_class, http_status, should_retry, test_full_refresh_config): response_mock = MagicMock() response_mock.status_code = http_status - stream = LeverHiringStream() + stream = LeverHiringStream(**test_full_refresh_config) assert stream.should_retry(response_mock) == should_retry -def test_backoff_time(patch_base_class): +def test_backoff_time(patch_base_class, test_full_refresh_config): response_mock = MagicMock() - stream = LeverHiringStream() + stream = LeverHiringStream(**test_full_refresh_config) expected_backoff_time = None assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile index c7f197999c7b..d77882fdd02d 100644 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ b/airbyte-integrations/connectors/source-mixpanel/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index 905e8270c0e5..6d226a9d03de 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -56,7 +56,10 @@ class MixpanelStream(HttpStream, ABC): send requests with planned delay: 3600/reqs_per_hour_limit seconds """ - url_base = "https://mixpanel.com/api/2.0/" + @property + def url_base(self): + prefix = "eu." if self.region == "EU" else "" + return f"https://{prefix}mixpanel.com/api/2.0/" # https://help.mixpanel.com/hc/en-us/articles/115004602563-Rate-Limits-for-Export-API-Endpoints#api-export-endpoint-rate-limits reqs_per_hour_limit = 400 # 1 req in 9 secs @@ -64,6 +67,7 @@ class MixpanelStream(HttpStream, ABC): def __init__( self, authenticator: HttpAuthenticator, + region: str = None, start_date: Union[date, str] = None, end_date: Union[date, str] = None, date_window_size: int = 30, # in days @@ -76,6 +80,7 @@ def __init__( self.date_window_size = date_window_size self.attribution_window = attribution_window self.additional_properties = select_properties_by_default + self.region = region if region else "US" super().__init__(authenticator=authenticator) @@ -112,6 +117,12 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp # wait for X seconds to match API limitations time.sleep(3600 / self.reqs_per_hour_limit) + def get_stream_params(self) -> Mapping[str, Any]: + """ + Fetch required parameters in a given stream. Used to create sub-streams + """ + return {"authenticator": self.authenticator, "region": self.region} + class IncrementalMixpanelStream(MixpanelStream, ABC): def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: @@ -229,7 +240,7 @@ def path(self, **kwargs) -> str: return "funnels" def funnel_slices(self, sync_mode) -> List[dict]: - funnel_slices = FunnelsList(authenticator=self.authenticator).read_records(sync_mode=sync_mode) + funnel_slices = FunnelsList(**self.get_stream_params()).read_records(sync_mode=sync_mode) funnel_slices = list(funnel_slices) # [{'funnel_id': , 'name': }, {...}] # save all funnels in dict(:, ...) @@ -523,7 +534,7 @@ def get_json_schema(self) -> Mapping[str, Any]: } # read existing Engage schema from API - schema_properties = EngageSchema(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) + schema_properties = EngageSchema(**self.get_stream_params()).read_records(sync_mode=SyncMode.full_refresh) for property_entry in schema_properties: property_name: str = property_entry["name"] property_type: str = property_entry["type"] @@ -553,7 +564,7 @@ def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: stream_slices = [] - cohorts = Cohorts(authenticator=self.authenticator).read_records(sync_mode=sync_mode) + cohorts = Cohorts(**self.get_stream_params()).read_records(sync_mode=sync_mode) for cohort in cohorts: stream_slices.append({"id": cohort["id"]}) @@ -692,7 +703,10 @@ class Export(DateSlicesMixin, IncrementalMixpanelStream): cursor_field = "time" reqs_per_hour_limit = 60 # 1 query per minute - url_base = "https://data.mixpanel.com/api/2.0/" + @property + def url_base(self): + prefix = "-eu" if self.region == "EU" else "" + return f"https://data{prefix}.mixpanel.com/api/2.0/" def path(self, **kwargs) -> str: return "export" @@ -716,6 +730,10 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp } } """ + if response.text == "terminated early\n": + # no data available + self.logger.warn(f"Couldn't fetch data from Export API. Response: {response.text}") + return [] for record_line in response.text.splitlines(): record = json.loads(record_line) @@ -758,7 +776,7 @@ def get_json_schema(self) -> Mapping[str, Any]: schema["additionalProperties"] = self.additional_properties # read existing Export schema from API - schema_properties = ExportSchema(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) + schema_properties = ExportSchema(**self.get_stream_params()).read_records(sync_mode=SyncMode.full_refresh) for property_entry in schema_properties: property_name: str = property_entry if property_name.startswith("$"): @@ -781,7 +799,7 @@ def __init__(self, token: str, auth_method: str = "Basic", **kwargs): class SourceMixpanel(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: """ See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 for an example. @@ -790,14 +808,15 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - authenticator = TokenAuthenticatorBase64(token=config["api_secret"]) + auth = TokenAuthenticatorBase64(token=config["api_secret"]) + funnels = FunnelsList(authenticator=auth, **config) try: response = requests.request( "GET", - url="https://mixpanel.com/api/2.0/funnels/list", + url=funnels.url_base + funnels.path(), headers={ "Accept": "application/json", - **authenticator.get_auth_header(), + **auth.get_auth_header(), }, ) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json index cead1e425fc1..d04cf04aeedf 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json @@ -38,6 +38,11 @@ "description": "The default value to use if no bookmark exists for an endpoint. Default is 1 year ago.", "examples": ["2021-11-16"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" + }, + "region": { + "type": "string", + "enum": ["US", "EU"], + "default": "US" } } } diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py index eccba1bcf421..7f49206280cb 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py @@ -32,12 +32,14 @@ def test_date_slices(): now = date.today() # Test with start_date now range - stream_slices = Annotations(authenticator=NoAuth(), start_date=now, end_date=now, date_window_size=1).stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - stream_slices = Annotations(authenticator=NoAuth(), start_date=now - timedelta(days=1), end_date=now, date_window_size=1).stream_slices( + stream_slices = Annotations(authenticator=NoAuth(), start_date=now, end_date=now, date_window_size=1, region="EU").stream_slices( sync_mode="any" ) + assert 1 == len(stream_slices) + + stream_slices = Annotations( + authenticator=NoAuth(), start_date=now - timedelta(days=1), end_date=now, date_window_size=1, region="US" + ).stream_slices(sync_mode="any") assert 2 == len(stream_slices) stream_slices = Annotations(authenticator=NoAuth(), start_date=now - timedelta(days=2), end_date=now, date_window_size=1).stream_slices( @@ -52,23 +54,40 @@ def test_date_slices(): # test with attribution_window stream_slices = Annotations( - authenticator=NoAuth(), start_date=now - timedelta(days=2), end_date=now, date_window_size=1, attribution_window=5 + authenticator=NoAuth(), + start_date=now - timedelta(days=2), + end_date=now, + date_window_size=1, + attribution_window=5, + region="US", ).stream_slices(sync_mode="any") assert 8 == len(stream_slices) # Test with start_date end_date range stream_slices = Annotations( - authenticator=NoAuth(), start_date=date.fromisoformat("2021-07-01"), end_date=date.fromisoformat("2021-07-01"), date_window_size=1 + authenticator=NoAuth(), + start_date=date.fromisoformat("2021-07-01"), + end_date=date.fromisoformat("2021-07-01"), + date_window_size=1, + region="US", ).stream_slices(sync_mode="any") assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}] == stream_slices stream_slices = Annotations( - authenticator=NoAuth(), start_date=date.fromisoformat("2021-07-01"), end_date=date.fromisoformat("2021-07-02"), date_window_size=1 + authenticator=NoAuth(), + start_date=date.fromisoformat("2021-07-01"), + end_date=date.fromisoformat("2021-07-02"), + date_window_size=1, + region="EU", ).stream_slices(sync_mode="any") assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}, {"start_date": "2021-07-02", "end_date": "2021-07-02"}] == stream_slices stream_slices = Annotations( - authenticator=NoAuth(), start_date=date.fromisoformat("2021-07-01"), end_date=date.fromisoformat("2021-07-03"), date_window_size=1 + authenticator=NoAuth(), + start_date=date.fromisoformat("2021-07-01"), + end_date=date.fromisoformat("2021-07-03"), + date_window_size=1, + region="US", ).stream_slices(sync_mode="any") assert [ {"start_date": "2021-07-01", "end_date": "2021-07-01"}, @@ -77,12 +96,19 @@ def test_date_slices(): ] == stream_slices stream_slices = Annotations( - authenticator=NoAuth(), start_date=date.fromisoformat("2021-07-01"), end_date=date.fromisoformat("2021-07-03"), date_window_size=2 + authenticator=NoAuth(), + start_date=date.fromisoformat("2021-07-01"), + end_date=date.fromisoformat("2021-07-03"), + date_window_size=2, + region="US", ).stream_slices(sync_mode="any") assert [{"start_date": "2021-07-01", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == stream_slices # test with stream_state stream_slices = Annotations( - authenticator=NoAuth(), start_date=date.fromisoformat("2021-07-01"), end_date=date.fromisoformat("2021-07-03"), date_window_size=1 + authenticator=NoAuth(), + start_date=date.fromisoformat("2021-07-01"), + end_date=date.fromisoformat("2021-07-03"), + date_window_size=1, ).stream_slices(sync_mode="any", stream_state={"date": "2021-07-02"}) assert [{"start_date": "2021-07-02", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == stream_slices diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index 0b9b95c55c0e..6e02f86d4e06 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.mongodb.MongoDbSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index b73819559a35..0683c7093134 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -8,5 +8,5 @@ COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar RUN tar xf ${APPLICATION}.tar --strip-components=1 -LABEL io.airbyte.version=0.3.5 +LABEL io.airbyte.version=0.3.6 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index f73af81f1a83..aa7dfb0068c1 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.mssql.MssqlSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 0f70ae3dd1bb..cf79f576aeff 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -39,6 +39,7 @@ import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.ssh.SshWrappedSource; import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.StateManager; @@ -71,6 +72,8 @@ public class MssqlSource extends AbstractJdbcSource implements Source { public static final String MSSQL_CDC_OFFSET = "mssql_cdc_offset"; public static final String MSSQL_DB_HISTORY = "mssql_db_history"; public static final String CDC_LSN = "_ab_cdc_lsn"; + public static final List HOST_KEY = List.of("host"); + public static final List PORT_KEY = List.of("port"); private final JdbcSourceOperations sourceOperations; @@ -326,7 +329,7 @@ private void readSsl(JsonNode sslMethod, List additionalParameters) { } public static void main(String[] args) throws Exception { - final Source source = new MssqlSource(); + final Source source = new SshWrappedSource(new MssqlSource(), HOST_KEY, PORT_KEY); LOGGER.info("starting source: {}", MssqlSource.class); new IntegrationRunner(source).run(args); LOGGER.info("completed source: {}", MssqlSource.class); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..7e31046ccd0e --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java @@ -0,0 +1,116 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.SyncMode; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public abstract class AbstractSshMssqlSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final String STREAM_NAME = "dbo.id_and_name"; + private static final String STREAM_NAME2 = "dbo.starships"; + + private JsonNode config; + + public abstract Path getConfigFilePath(); + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + config = Jsons.deserialize(IOs.readFile(getConfigFilePath())); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + + } + + @Override + protected String getImageName() { + return "airbyte/source-mssql:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.getSpecAndInjectSsh(); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME, + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME2, + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java index 95ee5274b18b..0683a910f374 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java @@ -28,9 +28,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; import io.airbyte.db.Databases; +import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; @@ -65,7 +65,7 @@ protected String getImageName() { @Override protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + return SshHelpers.getSpecAndInjectSsh(); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index 69926fab1cfd..5564ebf8c7a4 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -28,10 +28,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; import io.airbyte.db.Databases; +import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; @@ -94,7 +94,7 @@ protected String getImageName() { @Override protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + return SshHelpers.getSpecAndInjectSsh(); } @Override diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAnalyticsOauthFlow.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java similarity index 78% rename from airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAnalyticsOauthFlow.java rename to airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java index 7f7ba6ce9727..8479c2aba408 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAnalyticsOauthFlow.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java @@ -22,14 +22,15 @@ * SOFTWARE. */ -package io.airbyte.oauth.google; +package io.airbyte.integrations.source.mssql; -import io.airbyte.config.persistence.ConfigRepository; +import java.nio.file.Path; -public class GoogleAnalyticsOauthFlow extends GoogleOAuthFlow { +public class SshKeyMssqlSourceAcceptanceTest extends AbstractSshMssqlSourceAcceptanceTest { - public GoogleAnalyticsOauthFlow(ConfigRepository configRepository) { - super(configRepository, "https://www.googleapis.com/auth/analytics.readonly"); + @Override + public Path getConfigFilePath() { + return Path.of("secrets/ssh-key-config.json"); } } diff --git a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigSeedProvider.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java similarity index 61% rename from airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigSeedProvider.java rename to airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java index 8dfd747b252c..7c24a41de169 100644 --- a/airbyte-config/persistence/src/main/java/io/airbyte/config/persistence/ConfigSeedProvider.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java @@ -22,24 +22,15 @@ * SOFTWARE. */ -package io.airbyte.config.persistence; +package io.airbyte.integrations.source.mssql; -import io.airbyte.config.Configs; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import java.nio.file.Path; -public class ConfigSeedProvider { +public class SshPasswordMssqlSourceAcceptanceTest extends AbstractSshMssqlSourceAcceptanceTest { - private static final Logger LOGGER = LoggerFactory.getLogger(ConfigSeedProvider.class); - - public static ConfigPersistence get(Configs configs) { - if (FileSystemConfigPersistence.hasExistingConfigs(configs.getConfigRoot())) { - LOGGER.info("There is existing local config directory; seed from the config volume"); - return new FileSystemConfigPersistence(configs.getConfigRoot()); - } else { - LOGGER.info("There is no existing local config directory; seed from YAML files"); - return YamlSeedConfigPersistence.get(); - } + @Override + public Path getConfigFilePath() { + return Path.of("secrets/ssh-pwd-config.json"); } } diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 96d0f0473b63..507b996af9f2 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.mysql.MySqlSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 62f8da45b1a5..0e25bbadb877 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -38,6 +38,7 @@ import io.airbyte.db.jdbc.JdbcSourceOperations; import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.base.ssh.SshWrappedSource; import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.relationaldb.StateManager; @@ -246,7 +247,7 @@ public Set getExcludedInternalNameSpaces() { } public static void main(String[] args) throws Exception { - final Source source = new MySqlSource(); + final Source source = new SshWrappedSource(new MySqlSource(), List.of("host"), List.of("port")); LOGGER.info("starting source: {}", MySqlSource.class); new IntegrationRunner(source).run(args); LOGGER.info("completed source: {}", MySqlSource.class); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..7aa4c2847bb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractSshMySqlSourceAcceptanceTest.java @@ -0,0 +1,114 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.mysql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.ssh.SshHelpers; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.protocol.models.CatalogHelpers; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.protocol.models.DestinationSyncMode; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaPrimitive; +import io.airbyte.protocol.models.SyncMode; +import java.nio.file.Path; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public abstract class AbstractSshMySqlSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME2 = "starships"; + + private JsonNode config; + + public abstract Path getConfigFilePath(); + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + config = Jsons.deserialize(IOs.readFile(getConfigFilePath())); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + + } + + @Override + protected String getImageName() { + return "airbyte/source-mysql:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.getSpecAndInjectSsh(); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", config.get("database").asText(), STREAM_NAME), + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s.%s", config.get("database").asText(), STREAM_NAME2), + Field.of("id", JsonSchemaPrimitive.NUMBER), + Field.of("name", JsonSchemaPrimitive.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index c8acc8eb85dc..66ab4653cd70 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -28,9 +28,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; import io.airbyte.db.Databases; +import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -61,7 +61,7 @@ protected String getImageName() { @Override protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + return SshHelpers.getSpecAndInjectSsh(); } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java index e81d7a32d19b..2f96c4353167 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java @@ -28,9 +28,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; import io.airbyte.db.Databases; +import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; @@ -103,7 +103,7 @@ protected String getImageName() { @Override protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + return SshHelpers.getSpecAndInjectSsh(); } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..f8f6fe83dab5 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java @@ -0,0 +1,36 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.mysql; + +import java.nio.file.Path; + +public class SshKeyMySqlSourceAcceptanceTest extends AbstractSshMySqlSourceAcceptanceTest { + + @Override + public Path getConfigFilePath() { + return Path.of("secrets/ssh-key-config.json"); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..dc34530bc9f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java @@ -0,0 +1,36 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.mysql; + +import java.nio.file.Path; + +public class SshPasswordMySqlSourceAcceptanceTest extends AbstractSshMySqlSourceAcceptanceTest { + + @Override + public Path getConfigFilePath() { + return Path.of("secrets/ssh-pwd-config.json"); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 29d2262605fa..2ea28e91c11a 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -24,14 +24,18 @@ package io.airbyte.integrations.source.mysql; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.protocol.models.ConnectorSpecification; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -40,6 +44,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { @@ -124,4 +129,12 @@ public JsonNode getConfig() { return Jsons.clone(config); } + @Test + void testSpec() throws Exception { + final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + + assertEquals(expected, actual); + } + } diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index 8e5cac949873..4c8f1a5969fa 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.oracle.OracleSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index cb32c7dde20b..28b4ee4ff064 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile index 43eafe69c3f2..2c4b16295b8e 100644 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ b/airbyte-integrations/connectors/source-recharge/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index b9e9e49552c9..8595868f33e2 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -12,12 +12,13 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json" + timeout_seconds: 1200 incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - addresses: [ "created_at" ] + timeout_seconds: 900 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1200 diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json index fbedef9735c1..ae9cea634a97 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/abnormal_state.json @@ -1,23 +1,23 @@ { "addresses": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "charges": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "customers": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "discounts": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "onetimes": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "orders": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" }, "subscriptions": { - "created_at": "2050-05-18T00:00:00" + "updated_at": "2050-05-18T00:00:00" } } diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json index f37ccc314ea2..e878274c1f9b 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/configured_catalog.json @@ -6,12 +6,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -19,12 +19,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -42,12 +42,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -55,12 +55,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -78,12 +78,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -91,12 +91,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -124,12 +124,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] } ] } diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json b/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json index abd316e47387..cf16288f6364 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/streams_with_output_records_catalog.json @@ -6,12 +6,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -19,12 +19,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -42,12 +42,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -55,12 +55,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -68,12 +68,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] }, { "stream": { @@ -101,12 +101,12 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created_at"], + "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["created_at"] + "cursor_field": ["updated_at"] } ] } diff --git a/airbyte-integrations/connectors/source-recharge/requirements.txt b/airbyte-integrations/connectors/source-recharge/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-recharge/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json b/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json index 47f6bb771a5e..f667a07e67b9 100644 --- a/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json +++ b/airbyte-integrations/connectors/source-recharge/sample_files/sample_state.json @@ -1,23 +1,23 @@ { "addresses": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "charges": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "customers": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "discounts": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "onetimes": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "orders": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" }, "subscriptions": { - "created_at": "2021-04-02T00:00:00" + "updated_at": "2021-04-02T00:00:00" } } diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index d4c2222d4ae9..e7a200eba0b5 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -79,7 +79,7 @@ def get_stream_data(self, response_data: Any) -> List[dict]: class IncrementalRechargeStream(RechargeStream, ABC): - cursor_field = "created_at" + cursor_field = "updated_at" def __init__(self, start_date, **kwargs): super().__init__(**kwargs) diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index 9eabbe126f03..2df07034496c 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.redshift.RedshiftSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } repositories { diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index 1bd04b48cd16..aad0c3fbc2e6 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_bulk.json b/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_bulk.json index 7638681d8d3a..69da9893a876 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_bulk.json +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_bulk.json @@ -79,6 +79,42 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "LoginGeo", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["SystemModstamp"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "LoginHistory", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["LoginTime"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "PermissionSetTabSetting", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["SystemModstamp"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_rest.json b/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_rest.json index 08e80fbcecaf..bdf4425c618f 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_rest.json +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/configured_catalog_rest.json @@ -69,6 +69,42 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "LoginGeo", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["SystemModstamp"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "LoginHistory", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["LoginTime"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "PermissionSetTabSetting", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["SystemModstamp"], + "source_defined_primary_key": [["Id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json b/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json index d2ba96d22dd5..da64ffbef27e 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/future_state.json @@ -16,5 +16,14 @@ }, "ObjectPermissions": { "SystemModstamp": "2121-08-23T10:27:22.000Z" + }, + "LoginGeo": { + "SystemModstamp": "2121-08-23T10:27:22.000Z" + }, + "LoginHistory": { + "LoginTime": "2121-08-23T10:27:22.000Z" + }, + "PermissionSetTabSetting": { + "SystemModstamp": "2121-08-23T10:27:22.000Z" } } diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json index 0cd3961fc0b5..a1e03b234bca 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/sample_state.json @@ -13,5 +13,14 @@ }, "ObjectPermissions": { "SystemModstamp": "2021-08-23T10:27:22.000Z" + }, + "LoginGeo": { + "SystemModstamp": "2021-08-23T10:27:22.000Z" + }, + "LoginHistory": { + "LoginTime": "2021-08-23T10:27:22.000Z" + }, + "PermissionSetTabSetting": { + "SystemModstamp": "2021-08-23T10:27:22.000Z" } } diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 8616e625d4c3..3429fedf4baa 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -40,7 +40,7 @@ class SalesforceStream(HttpStream, ABC): - limit = 2000 + page_size = 2000 def __init__(self, sf_api: Salesforce, pk: str, stream_name: str, schema: dict = None, **kwargs): super().__init__(**kwargs) @@ -66,8 +66,8 @@ def path(self, **kwargs) -> str: def next_page_token(self, response: requests.Response) -> str: response_data = response.json() - if len(response_data["records"]) == self.limit and self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: - return f"WHERE {self.primary_key} > '{response_data['records'][-1][self.primary_key]}' " + if len(response_data["records"]) == self.page_size and self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: + return f"WHERE {self.primary_key} >= '{response_data['records'][-1][self.primary_key]}' " def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -91,7 +91,7 @@ def request_params( query += next_page_token if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: - query += f"ORDER BY {self.primary_key} ASC LIMIT {self.limit}" + query += f"ORDER BY {self.primary_key} ASC LIMIT {self.page_size}" return {"q": query} @@ -116,7 +116,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: class BulkSalesforceStream(SalesforceStream): - limit = 10000 + page_size = 30000 JOB_WAIT_TIMEOUT_MINS = 10 CHECK_INTERVAL_SECONDS = 2 @@ -186,7 +186,7 @@ def delete_job(self, url: str): def next_page_token(self, last_record: dict) -> str: if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: - return f"WHERE {self.primary_key} > '{last_record[self.primary_key]}' " + return f"WHERE {self.primary_key} >= '{last_record[self.primary_key]}' " def transform(self, record: dict, schema: dict = None): """ @@ -259,7 +259,7 @@ def read_records( for count, record in self.download_data(url=job_full_url): yield self.transform(record) - if count == self.limit: + if count == self.page_size: next_page_token = self.next_page_token(record) if not next_page_token: pagination_complete = True @@ -272,11 +272,12 @@ def read_records( if job_status in ["JobComplete", "Aborted", "Failed"]: self.delete_job(url=job_full_url) - pagination_complete = True + if job_status in ["Aborted", "Failed"]: + raise Exception(f"Job for {self.name} stream using BULK API was failed") class IncrementalSalesforceStream(SalesforceStream, ABC): - state_checkpoint_interval = 100 + state_checkpoint_interval = 500 def __init__(self, replication_key: str, start_date: str, **kwargs): super().__init__(**kwargs) @@ -285,7 +286,7 @@ def __init__(self, replication_key: str, start_date: str, **kwargs): def next_page_token(self, response: requests.Response) -> str: response_data = response.json() - if len(response_data["records"]) == self.limit and self.name not in UNSUPPORTED_FILTERING_STREAMS: + if len(response_data["records"]) == self.page_size and self.name not in UNSUPPORTED_FILTERING_STREAMS: return response_data["records"][-1][self.cursor_field] def request_params( @@ -304,9 +305,9 @@ def request_params( stream_date = stream_state.get(self.cursor_field) start_date = next_page_token or stream_date or self.start_date - query = f"SELECT {','.join(selected_properties.keys())} FROM {self.name} WHERE {self.cursor_field} > {start_date} " + query = f"SELECT {','.join(selected_properties.keys())} FROM {self.name} WHERE {self.cursor_field} >= {start_date} " if self.name not in UNSUPPORTED_FILTERING_STREAMS: - query += f"ORDER BY {self.cursor_field} ASC LIMIT {self.limit}" + query += f"ORDER BY {self.cursor_field} ASC LIMIT {self.page_size}" return {"q": query} @property diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 2e1f154b1d55..5631218d7bd8 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index ab0170e8b434..a8b2ef8d61a2 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -13,7 +13,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" timeout_seconds: 1200 - incremental: + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index 9ae53fdf1b45..a6b169a7e2dd 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -23,10 +23,10 @@ "custom_collections": { "updated_at": "2024-07-19T07:01:37-07:00" }, - "order_refunds": { + "orders_refunds": { "created_at": "2024-07-19T06:41:47-07:00" }, - "order_risks": { + "orders_risks": { "id": 9991307599038 }, "transactions": { diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json index 382e9f6e21e0..ef348454eb08 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -98,7 +98,7 @@ }, { "stream": { - "name": "order_refunds", + "name": "orders_refunds", "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, @@ -110,7 +110,7 @@ }, { "stream": { - "name": "order_risks", + "name": "orders_risks", "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index 8d3d1bbaae14..575f2171d4b3 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -1,15 +1,15 @@ { "customers": { - "updated_at": "2021-09-09T02:57:47-07:00" + "updated_at": "2021-09-19T09:08:24-07:00" }, "orders": { - "updated_at": "2021-09-09T02:57:43-07:00" + "updated_at": "2021-09-19T09:08:24-07:00" }, "draft_orders": { "updated_at": "2021-07-07T08:18:58-07:00" }, "products": { - "updated_at": "2021-08-18T02:40:22-07:00" + "updated_at": "2021-09-19T09:10:43-07:00" }, "abandoned_checkouts": { "updated_at": "2021-07-08T05:41:47-07:00" @@ -23,11 +23,11 @@ "custom_collections": { "updated_at": "2021-08-18T02:39:34-07:00" }, - "order_refunds": { + "orders_refunds": { "created_at": "2021-09-09T02:57:43-07:00" }, - "order_risks": { - "id": 6161307599037 + "orders_risks": { + "id": 6161307599036 }, "transactions": { "created_at": "2021-09-09T02:57:43-07:00" @@ -36,9 +36,9 @@ "updated_at": "2021-07-08T05:24:10-07:00" }, "price_rules": { - "updated_at": "2021-07-08T05:57:04-07:00" + "updated_at": "2021-09-10T06:48:10-07:00" }, "discount_codes": { - "updated_at": "2021-07-08T05:40:37-07:00" + "updated_at": "2021-09-10T06:48:10-07:00" } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json index 7cb02047ef54..3428f978b90b 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json @@ -117,7 +117,7 @@ "properties": { "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -143,7 +143,7 @@ "type": ["null", "array"] }, "total_line_items_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "closed_at": { "type": ["null", "string"], @@ -165,10 +165,10 @@ "type": ["null", "string"] }, "total_tax": { - "type": ["null", "string"] + "type": ["null", "number"] }, "subtotal_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "line_items": { "items": { @@ -181,7 +181,7 @@ "type": ["null", "integer"] }, "compare_at_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "destination_location_id": { "type": ["null", "integer"] @@ -190,7 +190,7 @@ "type": ["null", "string"] }, "line_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "origin_location_id": { "type": ["null", "integer"] @@ -237,7 +237,7 @@ }, "amount_set": {}, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -283,7 +283,7 @@ "properties": { "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -338,7 +338,7 @@ "type": ["null", "object"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "requires_shipping": { "type": ["null", "boolean"] @@ -399,7 +399,7 @@ "type": ["null", "string"] }, "total_discounts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "note": { "type": ["null", "string"] @@ -430,7 +430,7 @@ "type": ["null", "integer"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "requested_fulfillment_service_id": { "type": ["null", "string"] @@ -446,7 +446,7 @@ "properties": { "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -630,7 +630,7 @@ "type": ["null", "boolean"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] @@ -746,7 +746,7 @@ } }, "total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "cart_token": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json index 648b89423bb1..4a6c9a1b1219 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json @@ -79,7 +79,7 @@ "type": ["null", "boolean"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json index 3b2c670c487c..888b8a84e984 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json @@ -42,7 +42,7 @@ "type": ["null", "string"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] @@ -359,7 +359,7 @@ } }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -368,7 +368,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -379,7 +379,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -413,7 +413,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discount_set": { "type": ["null", "object"], @@ -422,7 +422,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -433,7 +433,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -460,7 +460,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -469,7 +469,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -480,7 +480,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -515,7 +515,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "rate": { "type": ["null", "number"] @@ -530,7 +530,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -541,7 +541,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -559,7 +559,7 @@ "discounted_price_set": {}, "price_set": {}, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -614,7 +614,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "rate": { "type": ["null", "number"] @@ -629,7 +629,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -640,7 +640,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -662,7 +662,7 @@ "type": ["null", "boolean"] }, "total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "completed_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json index 757084df0dfe..537e2ad5b1f3 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json @@ -1,4 +1,5 @@ { + "type": "object", "properties": { "id": { "type": ["null", "integer"] @@ -72,7 +73,7 @@ "type": ["null", "string"] }, "current_subtotal_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_subtotal_price_set": { "type": ["null", "object"], @@ -81,7 +82,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -92,7 +93,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -102,7 +103,7 @@ } }, "current_total_discounts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_total_discounts_set": { "type": ["null", "object"], @@ -111,7 +112,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -122,7 +123,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -135,7 +136,7 @@ "type": ["null", "string"] }, "current_total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_total_price_set": { "type": ["null", "object"], @@ -144,7 +145,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -155,7 +156,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -165,7 +166,7 @@ } }, "current_total_tax": { - "type": ["null", "string"] + "type": ["null", "number"] }, "current_total_tax_set": { "type": ["null", "object"], @@ -174,7 +175,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -185,7 +186,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -290,7 +291,7 @@ "type": ["null", "string"] }, "subtotal_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "subtotal_price_set": { "type": ["null", "object"], @@ -299,7 +300,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -310,7 +311,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -328,7 +329,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "rate": { "type": ["null", "number"] @@ -343,7 +344,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -354,7 +355,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -376,7 +377,7 @@ "type": ["null", "string"] }, "total_discounts": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discounts_set": { "type": ["null", "object"], @@ -385,7 +386,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -396,7 +397,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -406,7 +407,7 @@ } }, "total_line_items_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_line_items_price_set": { "type": ["null", "object"], @@ -415,7 +416,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -426,7 +427,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -436,10 +437,10 @@ } }, "total_outstanding": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_price_set": { "type": ["null", "object"], @@ -448,7 +449,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -459,7 +460,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -469,7 +470,7 @@ } }, "total_price_usd": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_shipping_price_set": { "type": ["null", "object"], @@ -478,7 +479,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -489,7 +490,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -499,7 +500,7 @@ } }, "total_tax": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_tax_set": { "type": ["null", "object"], @@ -508,7 +509,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -519,7 +520,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -529,7 +530,7 @@ } }, "total_tip_received": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_weight": { "type": ["null", "integer"] @@ -624,7 +625,7 @@ "type": ["null", "string"] }, "total_spent": { - "type": ["null", "string"] + "type": ["null", "number"] }, "last_order_id": { "type": ["null", "integer"] @@ -869,7 +870,7 @@ } }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -878,7 +879,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -889,7 +890,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -898,6 +899,9 @@ } } }, + "pre_tax_price": { + "type": ["null", "number"] + }, "product_exists": { "type": ["null", "boolean"] }, @@ -923,7 +927,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discount_set": { "type": ["null", "object"], @@ -932,7 +936,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -943,7 +947,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -970,7 +974,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -979,7 +983,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -990,7 +994,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1232,7 +1236,7 @@ } }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -1241,7 +1245,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1252,7 +1256,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1261,6 +1265,9 @@ } } }, + "pre_tax_price": { + "type": ["null", "number"] + }, "product_exists": { "type": ["null", "boolean"] }, @@ -1286,7 +1293,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "total_discount_set": { "type": ["null", "object"], @@ -1295,7 +1302,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1306,7 +1313,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1333,7 +1340,7 @@ "type": ["null", "object"], "properties": { "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -1342,7 +1349,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1353,7 +1360,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1595,7 +1602,7 @@ "type": ["null", "string"] }, "discounted_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "discounted_price_set": { "type": ["null", "object"], @@ -1604,7 +1611,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1615,7 +1622,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1628,7 +1635,7 @@ "type": ["null", "string"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "price_set": { "type": ["null", "object"], @@ -1637,7 +1644,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -1648,7 +1655,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_refunds.json similarity index 92% rename from airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json rename to airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_refunds.json index ab5f611279d5..e852ef31d321 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_refunds.json @@ -70,7 +70,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -81,7 +81,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -97,7 +97,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -108,7 +108,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -134,7 +134,7 @@ "type": ["null", "boolean"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "tax_lines": { "type": ["null", "array"], @@ -148,7 +148,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -159,7 +159,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -168,7 +168,7 @@ "type": ["null", "object"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "title": { "type": ["null", "string"] @@ -226,7 +226,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -237,7 +237,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -249,7 +249,7 @@ "type": ["null", "string"] }, "pre_tax_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "variant_title": { "type": ["null", "string"] @@ -262,7 +262,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -273,7 +273,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -286,7 +286,7 @@ "items": { "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "amount_set": { "properties": { @@ -296,7 +296,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -307,7 +307,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -330,7 +330,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -341,7 +341,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] } }, "type": ["null", "object"] @@ -359,7 +359,7 @@ "type": ["null", "string"] }, "total_discount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_risks.json similarity index 94% rename from airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json rename to airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_risks.json index 61cfde09f464..fae5bb5acf8f 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_risks.json @@ -14,7 +14,7 @@ "type": ["null", "string"] }, "score": { - "type": ["null", "string"] + "type": ["null", "number"] }, "recommendation": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json index 0fb5b34d67b7..ac2f37095ef3 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json @@ -173,7 +173,7 @@ "type": ["null", "integer"] }, "price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "image_id": { "type": ["null", "integer"] @@ -209,7 +209,7 @@ "type": ["null", "string"] }, "compare_at_price": { - "type": ["null", "string"] + "type": ["null", "number"] }, "updated_at": { "type": ["null", "string"], @@ -230,7 +230,7 @@ "type": ["null", "object"], "properties": { "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "currency_code": { "type": ["null", "string"] @@ -238,7 +238,7 @@ } }, "compare_at_price": { - "type": ["null", "string"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json index c89db3250bf4..32a9dfbb1503 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json @@ -22,7 +22,7 @@ "type": ["null", "integer"] }, "amount": { - "type": ["null", "string"] + "type": ["null", "number"] }, "authorization": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index eed41efeee80..fee4df8d88ac 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -34,6 +34,7 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from .transform import DataTypeEnforcer from .utils import EagerlyCachedStreamState as stream_state_cache from .utils import ShopifyRateLimiter as limiter @@ -51,6 +52,7 @@ class ShopifyStream(HttpStream, ABC): def __init__(self, config: Dict): super().__init__(authenticator=config["authenticator"]) + self._transformer = DataTypeEnforcer(self.get_json_schema()) self.config = config @property @@ -78,7 +80,11 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - yield from records + # transform method was implemented according to issue 4841 + # Shopify API returns price fields as a string and it should be converted to number + # this solution designed to convert string into number, but in future can be modified for general purpose + for record in records: + yield self._transformer.transform(record) @property @abstractmethod @@ -97,10 +103,10 @@ def state_checkpoint_interval(self) -> int: # Setting the default cursor field for all streams cursor_field = "updated_at" - @stream_state_cache.cache_stream_state def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} + @stream_state_cache.cache_stream_state def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) # If there is a next page token then we should only send pagination-related parameters. @@ -130,18 +136,6 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" -class OrderSubstream(IncrementalShopifyStream): - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - # get the last saved orders stream state - orders_stream = Orders(self.config) - orders_stream_state = stream_state_cache.cached_state.get(orders_stream.name) - for data in orders_stream.read_records(stream_state=orders_stream_state, **kwargs): - slice = super().read_records(stream_slice={"order_id": data["id"]}, **kwargs) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=slice) - - class Orders(IncrementalShopifyStream): data_field = "orders" @@ -157,6 +151,55 @@ def request_params( return params +class ChildSubstream(IncrementalShopifyStream): + + """ + ChildSubstream - provides slicing functionality for streams using parts of data from parent stream. + For example: + - `Refunds Orders` is the entity of `Orders`, + - `OrdersRisks` is the entity of `Orders`, + - `DiscountCodes` is the entity of `PriceRules`, etc. + + :: @ parent_stream_class - defines the parent stream object to read from + :: @ slice_key - defines the name of the property in stream slices dict. + :: @ record_field_name - the name of the field inside of parent stream record. Default is `id`. + """ + + parent_stream_class: object = None + slice_key: str = None + record_field_name: str = "id" + + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"limit": self.limit} + if next_page_token: + params.update(**next_page_token) + return params + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + Reading the parent stream for slices with structure: + EXAMPLE: for given record_field_name as `id` of Orders, + + Output: [ {slice_key: 123}, {slice_key: 456}, ..., {slice_key: 999} ] + """ + parent_stream = self.parent_stream_class(self.config) + parent_stream_state = stream_state_cache.cached_state.get(parent_stream.name) + for record in parent_stream.read_records(stream_state=parent_stream_state, **kwargs): + yield {self.slice_key: record[self.record_field_name]} + + def read_records( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + **kwargs, + ) -> Iterable[Mapping[str, Any]]: + """ Reading child streams records for each `id` """ + + self.logger.info(f"Reading {self.name} for {self.slice_key}: {stream_slice.get(self.slice_key)}") + records = super().read_records(stream_slice=stream_slice, **kwargs) + yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=records) + + class DraftOrders(IncrementalShopifyStream): data_field = "draft_orders" @@ -234,22 +277,26 @@ def request_params( return params -class OrderRefunds(OrderSubstream): +class OrdersRefunds(ChildSubstream): + + parent_stream_class: object = Orders + slice_key = "order_id" + data_field = "refunds" - order_field = "created_at" cursor_field = "created_at" - filter_field = "created_at_min" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: order_id = stream_slice["order_id"] return f"orders/{order_id}/{self.data_field}.json" -class OrderRisks(OrderSubstream): +class OrdersRisks(ChildSubstream): + + parent_stream_class: object = Orders + slice_key = "order_id" + data_field = "risks" - order_field = "id" cursor_field = "id" - filter_field = "since_id" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: order_id = stream_slice["order_id"] @@ -258,21 +305,14 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if not next_page_token and not stream_state: - params[self.filter_field] = 0 - return params +class Transactions(ChildSubstream): + + parent_stream_class: object = Orders + slice_key = "order_id" -class Transactions(OrderSubstream): data_field = "transactions" - order_field = "created_at" cursor_field = "created_at" - filter_field = "created_at_min" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: order_id = stream_slice["order_id"] @@ -293,23 +333,17 @@ def path(self, **kwargs) -> str: return f"{self.data_field}.json" -class DiscountCodes(IncrementalShopifyStream): +class DiscountCodes(ChildSubstream): + + parent_stream_class: object = PriceRules + slice_key = "price_rule_id" + data_field = "discount_codes" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: price_rule_id = stream_slice["price_rule_id"] return f"price_rules/{price_rule_id}/{self.data_field}.json" - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - # get the last saved price_rules stream state - price_rules_stream = PriceRules(self.config) - price_rules_stream_state = stream_state_cache.cached_state.get(price_rules_stream.name) - for data in price_rules_stream.read_records(stream_state=price_rules_stream_state, **kwargs): - slice = super().read_records(stream_slice={"price_rule_id": data["id"]}, **kwargs) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=slice) - class ShopifyAuthenticator(TokenAuthenticator): @@ -321,7 +355,6 @@ def get_auth_header(self) -> Mapping[str, Any]: return {"X-Shopify-Access-Token": f"{self._token}"} -# Basic Connections Check class SourceShopify(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: @@ -346,7 +379,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Mapping a input config of the user input configuration as defined in the connector spec. Defining streams to run. """ - + # config["ex_state"] = super().exposed_state config["authenticator"] = ShopifyAuthenticator(token=config["api_password"]) return [ @@ -358,8 +391,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Metafields(config), CustomCollections(config), Collects(config), - OrderRefunds(config), - OrderRisks(config), + OrdersRefunds(config), + OrdersRisks(config), Transactions(config), Pages(config), PriceRules(config), diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py new file mode 100644 index 000000000000..ad05dd5c0fa7 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py @@ -0,0 +1,121 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from decimal import Decimal +from typing import Any, Iterable, List, Mapping, MutableMapping + + +class DataTypeEnforcer: + """ + Transform class was implemented according to issue #4841 + Shopify API returns price fields as a string and it should be converted to the number + Some records fields contain objects and arrays, which contain price fields. + Those price fields should be transformed too. + This solution designed to convert string into number, but in future can be modified for general purpose + Correct types placed in schemes + Transformer iterates over records, compare values type with schema type and transform if it's needed + + Methods + ------- + _transform_array(self, array: List[Any], item_properties: Mapping[str, Any]) + Some fields type is array. Items inside array contain price fields, which should be transformed + This method iterate over items in array, compare schema types and convert if necessary + transform(self, field: Any, schema: Mapping[str, Any] = None) + Accepts field of Any type and schema, compere type of field and type in schema, convert if necessary + """ + + def __init__(self, schema: Mapping[str, Any], **kwargs): + super().__init__(**kwargs) + self._schema = schema + + @staticmethod + def _get_json_types(value_type: Any) -> List[str]: + json_types = { + str: ["string"], + int: ["integer", "number"], + float: ["number"], + dict: ["object"], + list: ["array"], + bool: ["boolean"], + type(None): [ + "null", + ], + } + return json_types.get(value_type) + + @staticmethod + def _types_from_schema(properties: Mapping[str, Any]) -> str: + schema_types = properties.get("type", []) + if not isinstance(schema_types, list): + schema_types = [ + schema_types, + ] + return schema_types + + @staticmethod + def _first_non_null_type(schema_types: List[str]) -> str: + not_null_types = schema_types.copy() + if "null" in not_null_types: + not_null_types.remove("null") + return not_null_types[0] + + @staticmethod + def _transform_number(value: Any): + return Decimal(value) + + def _transform_array(self, array: List[Any], item_properties: Mapping[str, Any]): + # iterate over items in array, compare schema types and convert if necessary. + for index, record in enumerate(array): + array[index] = self.transform(record, item_properties) + return array + + def _transform_object(self, record: MutableMapping[str, Any], properties: Mapping[str, Any]): + # compare schema types and convert if necessary. + for object_property, value in record.items(): + if value is None: + continue + if object_property in properties: + object_properties = properties.get(object_property) or {} + record[object_property] = self.transform(value, object_properties) + return record + + def transform(self, field: Any, schema: Mapping[str, Any] = None) -> Iterable[MutableMapping]: + schema = schema if schema is not None else self._schema + # get available types from schema + schema_types = self._types_from_schema(schema) + if schema_types and field is not None: + # if types presented in schema and field is not None, get available JSON Schema types for field + # and not null types from schema, check if field JSON Schema types presented in schema + field_json_types = self._get_json_types(type(field)) + schema_type = self._first_non_null_type(schema_types) + if not any(field_json_type in schema_types for field_json_type in field_json_types): + if schema_type == "number": + return self._transform_number(field) + if schema_type == "object": + properties = schema.get("properties", {}) + return self._transform_object(field, properties) + if schema_type == "array": + properties = schema.get("items", {}) + return self._transform_array(field, properties) + return field diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index 62346ba4381b..97aee0e45178 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -122,7 +122,7 @@ class EagerlyCachedStreamState: cached_state: Dict = {} @staticmethod - def stream_state_to_tmp(*args, state_object: Dict = cached_state) -> Dict: + def stream_state_to_tmp(*args, state_object: Dict = cached_state, **kwargs) -> Dict: """ Method to save the current stream state for future re-use within slicing. The method requires having the temporary `state_object` as placeholder. @@ -136,15 +136,12 @@ def stream_state_to_tmp(*args, state_object: Dict = cached_state) -> Dict: # Map the input *args, the sequece should be always keeped up to the input function # change the mapping if needed stream: object = args[0] # the self instance of the stream - current_stream_state: Dict = args[1] - latest_record: Dict = args[2] + current_stream_state: Dict = kwargs["stream_state"] or {} # get the current tmp_state_value tmp_stream_state_value = state_object.get(stream.name, {}).get(stream.cursor_field, "") - # Compare the `current_stream_state` with `latest_record` to have the initial state value + # Save the curent stream value for current sync, if present. if current_stream_state: - state_object[stream.name] = { - stream.cursor_field: min(current_stream_state.get(stream.cursor_field, ""), latest_record.get(stream.cursor_field, "")) - } + state_object[stream.name] = {stream.cursor_field: current_stream_state.get(stream.cursor_field, "")} # Check if we have the saved state and keep the minimun value if tmp_stream_state_value: state_object[stream.name] = { @@ -154,8 +151,8 @@ def stream_state_to_tmp(*args, state_object: Dict = cached_state) -> Dict: def cache_stream_state(func): @wraps(func) - def decorator(*args): - EagerlyCachedStreamState.stream_state_to_tmp(*args) - return func(*args) + def decorator(*args, **kwargs): + EagerlyCachedStreamState.stream_state_to_tmp(*args, **kwargs) + return func(*args, **kwargs) return decorator diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py index 0a05c0d31d39..833ad6875984 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py @@ -28,37 +28,35 @@ from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache # Define the Stream class for the test -STREAM = Orders(config={"authenticator": "token"}) +STREAM = Orders(config={"authenticator": None}) @pytest.mark.parametrize( - "stream, cur_stream_state, latest_record, state_object, expected_output", + "stream, cur_stream_state, state_object, expected_output", [ # When Full-Refresh: state_object: empty. - (STREAM, {STREAM.cursor_field: ""}, {STREAM.cursor_field: ""}, {}, {STREAM.name: {STREAM.cursor_field: ""}}), - (STREAM, {STREAM.cursor_field: ""}, {STREAM.cursor_field: "2021-01-01T01-01-01"}, {}, {STREAM.name: {STREAM.cursor_field: ""}}), + (STREAM, {STREAM.cursor_field: ""}, {}, {STREAM.name: {STREAM.cursor_field: ""}}), ], - ids=["Sync Started", "Sync in progress"], + ids=["Sync Started"], ) -def test_full_refresh(stream, cur_stream_state, latest_record, state_object, expected_output): +def test_full_refresh(stream, cur_stream_state, state_object, expected_output): """ When Sync = Full-Refresh: we don't have any state yet, so we need to keep the state_object at min value, thus empty. """ # create the fixure for *args based on input - args = [stream, cur_stream_state, latest_record] + args = [stream] # use the external tmp_state_object for this test - actual = stream_state_cache.stream_state_to_tmp(*args, state_object=state_object) + actual = stream_state_cache.stream_state_to_tmp(*args, state_object=state_object, stream_state=cur_stream_state) assert actual == expected_output @pytest.mark.parametrize( - "stream, cur_stream_state, latest_record, state_object, expected_output", + "stream, cur_stream_state, state_object, expected_output", [ # When start the incremental refresh, assuming we have the state of STREAM. ( STREAM, {STREAM.cursor_field: "2021-01-01T01-01-01"}, - {STREAM.cursor_field: "2021-01-05T02-02-02"}, {}, {STREAM.name: {STREAM.cursor_field: "2021-01-01T01-01-01"}}, ), @@ -66,19 +64,18 @@ def test_full_refresh(stream, cur_stream_state, latest_record, state_object, exp ( STREAM, {STREAM.cursor_field: "2021-01-05T02-02-02"}, - {STREAM.cursor_field: "2021-01-10T10-10-10"}, {}, {STREAM.name: {STREAM.cursor_field: "2021-01-05T02-02-02"}}, ), ], ids=["Sync Started", "Sync in progress"], ) -def test_incremental_sync(stream, cur_stream_state, latest_record, state_object, expected_output): +def test_incremental_sync(stream, cur_stream_state, state_object, expected_output): """ When Sync = Incremental Refresh: we already have the saved state from Full-Refresh sync, we have it passed as input to the Incremental Sync, so we need to back it up and reuse. """ # create the fixure for *args based on input - args = [stream, cur_stream_state, latest_record] - actual = stream_state_cache.stream_state_to_tmp(*args, state_object=state_object) + args = [stream] + actual = stream_state_cache.stream_state_to_tmp(*args, state_object=state_object, stream_state=cur_stream_state) assert actual == expected_output diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py new file mode 100644 index 000000000000..7f2e52af8855 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_transform.py @@ -0,0 +1,277 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from decimal import Decimal + +import pytest +from source_shopify.transform import DataTypeEnforcer + + +def find_by_path(path_list, value): + key_or_index = path_list.pop(0) + result_value = value[key_or_index] + if len(path_list) > 0: + result_value = find_by_path(path_list, result_value) + return result_value + + +def run_check(transform_object, schema, checks): + transformer = DataTypeEnforcer(schema) + transform_object = transformer.transform(transform_object) + for check in checks: + expected_value = find_by_path(check.get("path"), transform_object) + assert isinstance(expected_value, check["expected_type"]) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + {"id": 1}, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "integer"]}, + }, + }, + [ + { + "path": [ + "id", + ], + "expected_type": int, + }, + ], + ), + ( + { + "created_at": "2021-07-08T04:58:50-07:00", + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "string"]}, + }, + }, + [ + { + "path": [ + "created_at", + ], + "expected_type": str, + }, + ], + ), + ( + { + "note": None, + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "string"]}, + }, + }, + [ + { + "path": [ + "note", + ], + "expected_type": type(None), + }, + ], + ), + ( + { + "buyer_accepts_marketing": False, + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "boolean"]}, + }, + }, + [ + { + "path": [ + "buyer_accepts_marketing", + ], + "expected_type": bool, + }, + ], + ), + ], +) +def test_enforcer_correct_type(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "total_discounts": "0.00", + }, + { + "type": "object", + "properties": { + "total_discounts": {"type": ["null", "number"]}, + }, + }, + [ + { + "path": [ + "total_discounts", + ], + "expected_type": Decimal, + }, + ], + ), + ( + { + "total_price": "39.17", + }, + { + "type": "object", + "properties": { + "total_price": {"type": ["null", "number"]}, + }, + }, + [ + { + "path": [ + "total_price", + ], + "expected_type": Decimal, + }, + ], + ), + ], +) +def test_enforcer_string_to_number(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "customer": {"total_spent": "0.00"}, + }, + { + "type": "object", + "properties": { + "customer": {"type": "object", "properties": {"total_spent": {"type": ["null", "number"]}}}, + }, + }, + [ + {"path": ["customer", "total_spent"], "expected_type": Decimal}, + ], + ) + ], +) +def test_enforcer_nested_object(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "shipping_lines": [{"price": "20.17"}], + }, + { + "type": "object", + "properties": { + "shipping_lines": { + "items": {"properties": {"price": {"type": ["null", "number"]}}, "type": ["null", "object"]}, + "type": ["null", "array"], + }, + }, + }, + [ + {"path": ["shipping_lines", 0, "price"], "expected_type": Decimal}, + ], + ), + ( + { + "line_items": [{"gift_card": False, "grams": 112, "line_price": "19.00", "price": "19.00"}], + }, + { + "type": "object", + "properties": { + "line_items": { + "items": { + "properties": { + "grams": {"type": ["null", "integer"]}, + "line_price": {"type": ["null", "number"]}, + "gift_card": {"type": ["null", "boolean"]}, + "price": {"type": ["null", "number"]}, + }, + "type": ["null", "object"], + }, + "type": ["null", "array"], + }, + }, + }, + [ + {"path": ["line_items", 0, "line_price"], "expected_type": Decimal}, + ], + ), + ], +) +def test_enforcer_nested_array(transform_object, schema, checks): + run_check(transform_object, schema, checks) + + +@pytest.mark.parametrize( + "transform_object, schema, checks", + [ + ( + { + "line_items": ["19.00", "0.00"], + }, + { + "type": "object", + "properties": { + "line_items": { + "items": { + "type": ["null", "number"], + }, + "type": ["null", "array"], + }, + }, + }, + [ + {"path": ["line_items", 0], "expected_type": Decimal}, + {"path": ["line_items", 1], "expected_type": Decimal}, + ], + ), + ], +) +def test_enforcer_string_to_number_in_array(transform_object, schema, checks): + run_check(transform_object, schema, checks) diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle index 5ee5e68451b2..7648c5319a73 100644 --- a/airbyte-integrations/connectors/source-snowflake/build.gradle +++ b/airbyte-integrations/connectors/source-snowflake/build.gradle @@ -6,6 +6,7 @@ plugins { application { mainClass = 'io.airbyte.integrations.source.snowflake.SnowflakeSource' + applicationDefaultJvmArgs = ['-XX:MaxRAMPercentage=75.0'] } dependencies { diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index c77a33e1f2ee..904eb72c8594 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json index f297c1ea8cff..44b5c61a8527 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json @@ -18,8 +18,7 @@ "type": ["null", "string"] }, "redeem_by": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] }, "duration": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json index 77a558985a37..d97e5f7895f1 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json @@ -75,8 +75,7 @@ "type": ["null", "number"] }, "redeem_by": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] }, "duration_in_months": { "type": ["null", "number"] diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_config/StandardSourceDefinition.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_config/StandardSourceDefinition.yaml index 93cc33d9e619..c50d3ae1bb7b 100644 --- a/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_config/StandardSourceDefinition.yaml +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_config/StandardSourceDefinition.yaml @@ -10,7 +10,7 @@ required: - dockerRepository - dockerImageTag - documentationUrl -additionalProperties: false +additionalProperties: true properties: sourceDefinitionId: type: string diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_db/Attempts.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_db/Attempts.yaml index e978588d8a93..758f53c322f6 100644 --- a/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_db/Attempts.yaml +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_14_0/airbyte_db/Attempts.yaml @@ -11,7 +11,7 @@ required: - status - created_at - updated_at -additionalProperties: false +additionalProperties: true properties: id: type: number diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_20_0/StandardSourceDefinition.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_20_0/StandardSourceDefinition.yaml index 9afb01c5d426..a304f62c40dd 100644 --- a/airbyte-migration/src/main/resources/migrations/migrationV0_20_0/StandardSourceDefinition.yaml +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_20_0/StandardSourceDefinition.yaml @@ -10,7 +10,7 @@ required: - dockerRepository - dockerImageTag - documentationUrl -additionalProperties: false +additionalProperties: true properties: sourceDefinitionId: type: string diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_23_0/StandardSourceDefinition.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_23_0/StandardSourceDefinition.yaml index 9afb01c5d426..a304f62c40dd 100644 --- a/airbyte-migration/src/main/resources/migrations/migrationV0_23_0/StandardSourceDefinition.yaml +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_23_0/StandardSourceDefinition.yaml @@ -10,7 +10,7 @@ required: - dockerRepository - dockerImageTag - documentationUrl -additionalProperties: false +additionalProperties: true properties: sourceDefinitionId: type: string diff --git a/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationCurrentSchemaTest.java b/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationCurrentSchemaTest.java index f330df5c3b00..685f98de6c10 100644 --- a/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationCurrentSchemaTest.java +++ b/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationCurrentSchemaTest.java @@ -30,14 +30,11 @@ public class MigrationCurrentSchemaTest { - /** - * The file-based migration is deprecated. We need to ensure that v0.29.0 is the last one. All new - * migrations should be written in Flyway. - */ @Test public void testLastMigration() { final Migration lastMigration = Migrations.MIGRATIONS.get(Migrations.MIGRATIONS.size() - 1); - assertEquals(Migrations.MIGRATION_V_0_29_0.getVersion(), lastMigration.getVersion()); + assertEquals(Migrations.MIGRATION_V_0_29_0.getVersion(), lastMigration.getVersion(), + "The file-based migration is deprecated. Please do not write a new migration this way. Use Flyway instead."); } } diff --git a/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationIntegrationTest.java b/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationIntegrationTest.java index c8769f50b82c..275aba1c5a08 100644 --- a/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationIntegrationTest.java +++ b/airbyte-migration/src/test/java/io/airbyte/migrate/MigrationIntegrationTest.java @@ -144,8 +144,9 @@ void testMultipleMigrations() throws IOException { @Test void testInvalidInputRecord() throws IOException { - // attempt to input records that have foobar added. the input schema does NOT include foobar. - final Map> invalidInputRecords = addFooBarToAllRecordsExceptMetadata(V0_14_0_TEST_RECORDS); + // attempt to input records that miss sourceDefinitionId in standard source definition, which is + // required + final Map> invalidInputRecords = removeSourceDefinitionId(V0_14_0_TEST_RECORDS); writeInputArchive(inputRoot, invalidInputRecords, TEST_MIGRATIONS.get(0).getVersion()); final String targetVersion = TEST_MIGRATIONS.get(1).getVersion(); @@ -321,6 +322,21 @@ private static Map> addFooBarToAllRecordsExceptMetada .collect(Collectors.toList()))); } + private static Map> removeSourceDefinitionId(Map> records) { + return records.entrySet() + .stream() + .collect(Collectors.toMap(Entry::getKey, e -> e.getValue() + .stream() + .map(r -> { + final JsonNode expectedRecord = Jsons.clone(r); + if (expectedRecord.has("sourceDefinitionId")) { + ((ObjectNode) expectedRecord).remove("sourceDefinitionId"); + } + return expectedRecord; + }) + .collect(Collectors.toList()))); + } + private static void writeInputs(ResourceType resourceType, Set resourceNames, Path fileParent, diff --git a/airbyte-migration/src/test/resources/migrations/migrationV0_14_1/airbyte_config/StandardSourceDefinition.yaml b/airbyte-migration/src/test/resources/migrations/migrationV0_14_1/airbyte_config/StandardSourceDefinition.yaml index 38db6802b89b..12ac29c614b6 100644 --- a/airbyte-migration/src/test/resources/migrations/migrationV0_14_1/airbyte_config/StandardSourceDefinition.yaml +++ b/airbyte-migration/src/test/resources/migrations/migrationV0_14_1/airbyte_config/StandardSourceDefinition.yaml @@ -11,7 +11,7 @@ required: - dockerImageTag - documentationUrl - foo -additionalProperties: false +additionalProperties: true properties: sourceDefinitionId: type: string diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java similarity index 56% rename from airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleOAuthFlow.java rename to airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java index 94a6c3c296e1..43728c6562a8 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleOAuthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/BaseOAuthFlow.java @@ -22,24 +22,18 @@ * SOFTWARE. */ -package io.airbyte.oauth.google; +package io.airbyte.oauth; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableMap.Builder; import io.airbyte.commons.json.Jsons; import io.airbyte.config.ConfigSchema; import io.airbyte.config.DestinationOAuthParameter; import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.oauth.MoreOAuthParameters; -import io.airbyte.oauth.OAuthFlowImplementation; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.net.URI; -import java.net.URISyntaxException; import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpClient.Version; @@ -49,82 +43,65 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; -import org.apache.http.client.utils.URIBuilder; +import java.util.function.Supplier; +import org.apache.commons.lang3.RandomStringUtils; -/** - * Following docs from https://developers.google.com/identity/protocols/oauth2/web-server - */ -public class GoogleOAuthFlow implements OAuthFlowImplementation { +public abstract class BaseOAuthFlow implements OAuthFlowImplementation { private final HttpClient httpClient; - - private final static String CONSENT_URL = "https://accounts.google.com/o/oauth2/v2/auth"; - private final static String ACCESS_TOKEN_URL = "https://oauth2.googleapis.com/token"; - - private final String scope; - private final Map defaultQueryParams; - private final ConfigRepository configRepository; + private final Supplier stateSupplier; - public GoogleOAuthFlow(ConfigRepository configRepository, String scope) { - this(configRepository, scope, HttpClient.newBuilder().version(Version.HTTP_1_1).build()); + public BaseOAuthFlow(ConfigRepository configRepository) { + this(configRepository, HttpClient.newBuilder().version(Version.HTTP_1_1).build(), BaseOAuthFlow::generateRandomState); } - @VisibleForTesting - GoogleOAuthFlow(ConfigRepository configRepository, String scope, HttpClient httpClient) { + public BaseOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier stateSupplier) { this.configRepository = configRepository; this.httpClient = httpClient; - this.scope = scope; - this.defaultQueryParams = ImmutableMap.builder() - .put("scope", this.scope) - .put("access_type", "offline") - .put("include_granted_scopes", "true") - .put("response_type", "code") - .put("prompt", "consent") - .build(); - + this.stateSupplier = stateSupplier; } @Override public String getSourceConsentUrl(UUID workspaceId, UUID sourceDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException { final JsonNode oAuthParamConfig = getSourceOAuthParamConfig(workspaceId, sourceDefinitionId); - return getConsentUrl(sourceDefinitionId, getClientIdUnsafe(oAuthParamConfig), redirectUrl); + return formatConsentUrl(sourceDefinitionId, getClientIdUnsafe(oAuthParamConfig), redirectUrl); } @Override public String getDestinationConsentUrl(UUID workspaceId, UUID destinationDefinitionId, String redirectUrl) throws IOException, ConfigNotFoundException { final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, destinationDefinitionId); - return getConsentUrl(destinationDefinitionId, getClientIdUnsafe(oAuthParamConfig), redirectUrl); + return formatConsentUrl(destinationDefinitionId, getClientIdUnsafe(oAuthParamConfig), redirectUrl); } - private String getConsentUrl(UUID definitionId, String clientId, String redirectUrl) { - try { - URIBuilder uriBuilder = new URIBuilder(CONSENT_URL) - .addParameter("state", definitionId.toString()) - .addParameter("client_id", clientId) - .addParameter("redirect_uri", redirectUrl); - for (Map.Entry queryParameter : defaultQueryParams.entrySet()) { - uriBuilder.addParameter(queryParameter.getKey(), queryParameter.getValue()); - } - return uriBuilder.toString(); - } catch (URISyntaxException e) { - throw new IllegalArgumentException(e); - } + /** + * Depending on the OAuth flow implementation, the URL to grant user's consent may differ, + * especially in the query parameters to be provided. This function should generate such consent URL + * accordingly. + */ + protected abstract String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException; + + private static String generateRandomState() { + return RandomStringUtils.randomAlphanumeric(7); + } + + /** + * Generate a string to use as state in the OAuth process. + */ + protected String getState() { + return stateSupplier.get(); } @Override public Map completeSourceOAuth(UUID workspaceId, UUID sourceDefinitionId, Map queryParams, String redirectUrl) throws IOException, ConfigNotFoundException { - if (queryParams.containsKey("code")) { - final String code = (String) queryParams.get("code"); - final JsonNode oAuthParamConfig = getSourceOAuthParamConfig(workspaceId, sourceDefinitionId); - final String clientId = getClientIdUnsafe(oAuthParamConfig); - final String clientSecret = getClientSecretUnsafe(oAuthParamConfig); - return completeOAuthFlow(clientId, clientSecret, code, redirectUrl); - } else { - throw new IOException("Undefined 'code' from consent redirected url."); - } + final JsonNode oAuthParamConfig = getSourceOAuthParamConfig(workspaceId, sourceDefinitionId); + return completeOAuthFlow( + getClientIdUnsafe(oAuthParamConfig), + getClientSecretUnsafe(oAuthParamConfig), + extractCodeParameter(queryParams), + redirectUrl); } @Override @@ -133,57 +110,51 @@ public Map completeDestinationOAuth(UUID workspaceId, Map queryParams, String redirectUrl) throws IOException, ConfigNotFoundException { - if (queryParams.containsKey("code")) { - final String code = (String) queryParams.get("code"); - final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, destinationDefinitionId); - final String clientId = getClientIdUnsafe(oAuthParamConfig); - final String clientSecret = getClientSecretUnsafe(oAuthParamConfig); - return completeOAuthFlow(clientId, clientSecret, code, redirectUrl); - } else { - throw new IOException("Undefined 'code' from consent redirected url."); - } + final JsonNode oAuthParamConfig = getDestinationOAuthParamConfig(workspaceId, destinationDefinitionId); + return completeOAuthFlow( + getClientIdUnsafe(oAuthParamConfig), + getClientSecretUnsafe(oAuthParamConfig), + extractCodeParameter(queryParams), + redirectUrl); } - protected Map completeOAuthFlow(String clientId, String clientSecret, String code, String redirectUrl) throws IOException { - final ImmutableMap body = new Builder() - .put("client_id", clientId) - .put("client_secret", clientSecret) - .put("code", code) - .put("grant_type", "authorization_code") - .put("redirect_uri", redirectUrl) - .build(); + private Map completeOAuthFlow(String clientId, String clientSecret, String authCode, String redirectUrl) throws IOException { final HttpRequest request = HttpRequest.newBuilder() - .POST(HttpRequest.BodyPublishers.ofString(toUrlEncodedString(body))) - .uri(URI.create(ACCESS_TOKEN_URL)) + .POST(HttpRequest.BodyPublishers.ofString(toUrlEncodedString(getAccessTokenQueryParameters(clientId, clientSecret, authCode, redirectUrl)))) + .uri(URI.create(getAccessTokenUrl())) .header("Content-Type", "application/x-www-form-urlencoded") .build(); - final HttpResponse response; + // TODO: Handle error response to report better messages try { - response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - var data = Jsons.deserialize(response.body()); - if (data.has("refresh_token")) { - return Map.of("refresh_token", data.get("refresh_token").asText()); - } else { - // TODO This means the response from Google did not have a refresh token and is probably a - // programming error - // handle this better - throw new IOException(String.format("Missing 'refresh_token' in query params from %s. Response: %s", ACCESS_TOKEN_URL, data)); - } + final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());; + return extractRefreshToken(Jsons.deserialize(response.body())); } catch (InterruptedException e) { - throw new IOException("Failed to complete Google OAuth flow", e); + throw new IOException("Failed to complete OAuth flow", e); } } - private static String toUrlEncodedString(ImmutableMap body) { - final StringBuilder result = new StringBuilder(); - for (var entry : body.entrySet()) { - if (result.length() > 0) { - result.append("&"); - } - result.append(entry.getKey()).append("=").append(entry.getValue()); - } - return result.toString(); - } + /** + * Once the user is redirected after getting their consent, the API should redirect them to a + * specific redirection URL along with query parameters. This function should parse and extract the + * code from these query parameters in order to continue the OAuth Flow. + */ + protected abstract String extractCodeParameter(Map queryParams) throws IOException; + + /** + * Returns the URL where to retrieve the access token from. + */ + protected abstract String getAccessTokenUrl(); + + /** + * Query parameters to provide the access token url with. + */ + protected abstract Map getAccessTokenQueryParameters(String clientId, String clientSecret, String authCode, String redirectUrl); + + /** + * Once the auth code is exchange for a refresh token, the oauth flow implementation can extract and + * returns the values of fields to be used in the connector's configurations. + */ + protected abstract Map extractRefreshToken(JsonNode data) throws IOException; private JsonNode getSourceOAuthParamConfig(UUID workspaceId, UUID sourceDefinitionId) throws IOException, ConfigNotFoundException { try { @@ -213,7 +184,7 @@ private JsonNode getDestinationOAuthParamConfig(UUID workspaceId, UUID destinati } } - private String UrlEncode(String s) { + private static String urlEncode(String s) { try { return URLEncoder.encode(s, StandardCharsets.UTF_8); } catch (Exception e) { @@ -231,7 +202,7 @@ protected String getClientIdUnsafe(JsonNode oauthConfig) { if (oauthConfig.get("client_id") != null) { return oauthConfig.get("client_id").asText(); } else { - throw new IllegalArgumentException("Undefined parameter 'client_id' for Google OAuth Flow."); + throw new IllegalArgumentException("Undefined parameter 'client_id' necessary for the OAuth Flow."); } } @@ -245,8 +216,19 @@ protected String getClientSecretUnsafe(JsonNode oauthConfig) { if (oauthConfig.get("client_secret") != null) { return oauthConfig.get("client_secret").asText(); } else { - throw new IllegalArgumentException("Undefined parameter 'client_secret' for Google OAuth Flow."); + throw new IllegalArgumentException("Undefined parameter 'client_secret' necessary for the OAuth Flow."); } } + private static String toUrlEncodedString(Map body) { + final StringBuilder result = new StringBuilder(); + for (var entry : body.entrySet()) { + if (result.length() > 0) { + result.append("&"); + } + result.append(entry.getKey()).append("=").append(urlEncode(entry.getValue())); + } + return result.toString(); + } + } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 645b5296b243..dfd9953b9944 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -26,8 +26,10 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.oauth.google.GoogleAdsOauthFlow; -import io.airbyte.oauth.google.GoogleAnalyticsOauthFlow; +import io.airbyte.oauth.flows.FacebookMarketingOAuthFlow; +import io.airbyte.oauth.flows.google.GoogleAdsOAuthFlow; +import io.airbyte.oauth.flows.google.GoogleAnalyticsOAuthFlow; +import io.airbyte.oauth.flows.google.GoogleSearchConsoleOAuthFlow; import java.util.Map; public class OAuthImplementationFactory { @@ -36,8 +38,10 @@ public class OAuthImplementationFactory { public OAuthImplementationFactory(ConfigRepository configRepository) { OAUTH_FLOW_MAPPING = ImmutableMap.builder() - .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOauthFlow(configRepository)) - .put("airbyte/source-google-ads", new GoogleAdsOauthFlow(configRepository)) + .put("airbyte/source-facebook-marketing", new FacebookMarketingOAuthFlow(configRepository)) + .put("airbyte/source-google-ads", new GoogleAdsOAuthFlow(configRepository)) + .put("airbyte/source-google-analytics-v4", new GoogleAnalyticsOAuthFlow(configRepository)) + .put("airbyte/source-google-search-console", new GoogleSearchConsoleOAuthFlow(configRepository)) .build(); } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java new file mode 100644 index 000000000000..c15593bc90fc --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlow.java @@ -0,0 +1,113 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuthFlow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from + * https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow + */ +public class FacebookMarketingOAuthFlow extends BaseOAuthFlow { + + private static final String ACCESS_TOKEN_URL = "https://graph.facebook.com/v11.0/oauth/access_token"; + + public FacebookMarketingOAuthFlow(ConfigRepository configRepository) { + super(configRepository); + } + + @VisibleForTesting + FacebookMarketingOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException { + final URIBuilder builder = new URIBuilder() + .setScheme("https") + .setHost("www.facebook.com") + .setPath("v11.0/dialog/oauth") + // required + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("state", getState()) + // optional + .addParameter("response_type", "code") + .addParameter("scope", "ads_management,ads_read,read_insights"); + try { + return builder.build().toString(); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected String extractCodeParameter(Map queryParams) throws IOException { + if (queryParams.containsKey("code")) { + return (String) queryParams.get("code"); + } else { + throw new IOException("Undefined 'code' from consent redirected url."); + } + } + + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, String clientSecret, String authCode, String redirectUrl) { + return ImmutableMap.builder() + // required + .put("client_id", clientId) + .put("redirect_uri", redirectUrl) + .put("client_secret", clientSecret) + .put("code", authCode) + .build(); + } + + @Override + protected Map extractRefreshToken(JsonNode data) throws IOException { + // Facebook does not have refresh token but calls it "long lived access token" instead: + // see https://developers.facebook.com/docs/facebook-login/access-tokens/refreshing + if (data.has("access_token")) { + return Map.of("access_token", data.get("access_token").asText()); + } else { + throw new IOException(String.format("Missing 'access_token' in query params from %s", ACCESS_TOKEN_URL)); + } + } + +} diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAdsOauthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlow.java similarity index 75% rename from airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAdsOauthFlow.java rename to airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlow.java index 8a26b6e2ecb5..4aa8d33a9dc0 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/google/GoogleAdsOauthFlow.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlow.java @@ -22,7 +22,7 @@ * SOFTWARE. */ -package io.airbyte.oauth.google; +package io.airbyte.oauth.flows.google; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -31,19 +31,25 @@ import java.io.IOException; import java.net.http.HttpClient; import java.util.Map; +import java.util.function.Supplier; -public class GoogleAdsOauthFlow extends GoogleOAuthFlow { +public class GoogleAdsOAuthFlow extends GoogleOAuthFlow { @VisibleForTesting - static final String SCOPE = "https://www.googleapis.com/auth/adwords"; + static final String SCOPE_URL = "https://www.googleapis.com/auth/adwords"; - public GoogleAdsOauthFlow(ConfigRepository configRepository) { - super(configRepository, SCOPE); + public GoogleAdsOAuthFlow(ConfigRepository configRepository) { + super(configRepository); } @VisibleForTesting - GoogleAdsOauthFlow(ConfigRepository configRepository, HttpClient client) { - super(configRepository, SCOPE, client); + GoogleAdsOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String getScope() { + return SCOPE_URL; } @Override @@ -61,9 +67,9 @@ protected String getClientSecretUnsafe(JsonNode config) { } @Override - protected Map completeOAuthFlow(String clientId, String clientSecret, String code, String redirectUrl) throws IOException { + protected Map extractRefreshToken(JsonNode data) throws IOException { // the config object containing refresh token is nested inside the "credentials" object - return Map.of("credentials", super.completeOAuthFlow(clientId, clientSecret, code, redirectUrl)); + return Map.of("credentials", super.extractRefreshToken(data)); } } diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlow.java new file mode 100644 index 000000000000..b3b702565762 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlow.java @@ -0,0 +1,74 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.airbyte.config.persistence.ConfigRepository; +import java.io.IOException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.function.Supplier; + +public class GoogleAnalyticsOAuthFlow extends GoogleOAuthFlow { + + public static final String SCOPE_URL = "https://www.googleapis.com/auth/analytics.readonly"; + + public GoogleAnalyticsOAuthFlow(ConfigRepository configRepository) { + super(configRepository); + } + + @VisibleForTesting + GoogleAnalyticsOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String getScope() { + return SCOPE_URL; + } + + @Override + protected String getClientIdUnsafe(JsonNode config) { + // the config object containing client ID and secret is nested inside the "credentials" object + Preconditions.checkArgument(config.hasNonNull("credentials")); + return super.getClientIdUnsafe(config.get("credentials")); + } + + @Override + protected String getClientSecretUnsafe(JsonNode config) { + // the config object containing client ID and secret is nested inside the "credentials" object + Preconditions.checkArgument(config.hasNonNull("credentials")); + return super.getClientSecretUnsafe(config.get("credentials")); + } + + @Override + protected Map extractRefreshToken(JsonNode data) throws IOException { + // the config object containing refresh token is nested inside the "credentials" object + return Map.of("credentials", super.extractRefreshToken(data)); + } + +} diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java new file mode 100644 index 000000000000..dd71288b6194 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleOAuthFlow.java @@ -0,0 +1,125 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuthFlow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from https://developers.google.com/identity/protocols/oauth2/web-server + */ +public abstract class GoogleOAuthFlow extends BaseOAuthFlow { + + private static final String ACCESS_TOKEN_URL = "https://oauth2.googleapis.com/token"; + + public GoogleOAuthFlow(ConfigRepository configRepository) { + super(configRepository); + } + + @VisibleForTesting + GoogleOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(UUID definitionId, String clientId, String redirectUrl) throws IOException { + final URIBuilder builder = new URIBuilder() + .setScheme("https") + .setHost("accounts.google.com") + .setPath("o/oauth2/v2/auth") + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("response_type", "code") + .addParameter("scope", getScope()) + // recommended + .addParameter("access_type", "offline") + .addParameter("state", getState()) + // optional + .addParameter("include_granted_scopes", "true") + // .addParameter("login_hint", "user_email") + .addParameter("prompt", "consent"); + try { + return builder.build().toString(); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + /** + * @return the scope for the specific google oauth implementation. + */ + protected abstract String getScope(); + + @Override + protected String extractCodeParameter(Map queryParams) throws IOException { + if (queryParams.containsKey("code")) { + return (String) queryParams.get("code"); + } else { + throw new IOException("Undefined 'code' from consent redirected url."); + } + } + + @Override + protected String getAccessTokenUrl() { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, String clientSecret, String authCode, String redirectUrl) { + return ImmutableMap.builder() + .put("client_id", clientId) + .put("client_secret", clientSecret) + .put("code", authCode) + .put("grant_type", "authorization_code") + .put("redirect_uri", redirectUrl) + .build(); + } + + @Override + protected Map extractRefreshToken(JsonNode data) throws IOException { + final Map result = new HashMap<>(); + if (data.has("access_token")) { + result.put("access_token", data.get("access_token").asText()); + } + if (data.has("refresh_token")) { + result.put("refresh_token", data.get("refresh_token").asText()); + } else { + throw new IOException(String.format("Missing 'refresh_token' in query params from %s", ACCESS_TOKEN_URL)); + } + return result; + } + +} diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java new file mode 100644 index 000000000000..8eacaa00a72f --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlow.java @@ -0,0 +1,75 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import io.airbyte.config.persistence.ConfigRepository; +import java.io.IOException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.function.Supplier; + +public class GoogleSearchConsoleOAuthFlow extends GoogleOAuthFlow { + + @VisibleForTesting + static final String SCOPE_URL = "https://www.googleapis.com/auth/webmasters.readonly"; + + public GoogleSearchConsoleOAuthFlow(ConfigRepository configRepository) { + super(configRepository); + } + + @VisibleForTesting + GoogleSearchConsoleOAuthFlow(ConfigRepository configRepository, HttpClient httpClient, Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String getScope() { + return SCOPE_URL; + } + + @Override + protected String getClientIdUnsafe(JsonNode config) { + // the config object containing client ID and secret is nested inside the "authorization" object + Preconditions.checkArgument(config.hasNonNull("authorization")); + return super.getClientIdUnsafe(config.get("authorization")); + } + + @Override + protected String getClientSecretUnsafe(JsonNode config) { + // the config object containing client ID and secret is nested inside the "authorization" object + Preconditions.checkArgument(config.hasNonNull("authorization")); + return super.getClientSecretUnsafe(config.get("authorization")); + } + + @Override + protected Map extractRefreshToken(JsonNode data) throws IOException { + // the config object containing refresh token is nested inside the "authorization" object + return Map.of("authorization", super.extractRefreshToken(data)); + } + +} diff --git a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth/GoogleOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/FacebookMarketingOAuthFlowIntegrationTest.java similarity index 89% rename from airbyte-oauth/src/test-integration/java/io.airbyte.oauth/GoogleOAuthFlowIntegrationTest.java rename to airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/FacebookMarketingOAuthFlowIntegrationTest.java index 55ee5f99e91d..db537cad8139 100644 --- a/airbyte-oauth/src/test-integration/java/io.airbyte.oauth/GoogleOAuthFlowIntegrationTest.java +++ b/airbyte-oauth/src/test-integration/java/io.airbyte.oauth.flows/FacebookMarketingOAuthFlowIntegrationTest.java @@ -22,7 +22,7 @@ * SOFTWARE. */ -package io.airbyte.oauth; +package io.airbyte.oauth.flows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -37,8 +37,6 @@ import io.airbyte.config.SourceOAuthParameter; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.oauth.google.GoogleAnalyticsOauthFlow; -import io.airbyte.oauth.google.GoogleOAuthFlow; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.io.OutputStream; @@ -55,14 +53,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class GoogleOAuthFlowIntegrationTest { +public class FacebookMarketingOAuthFlowIntegrationTest { - private static final Logger LOGGER = LoggerFactory.getLogger(GoogleOAuthFlowIntegrationTest.class); + private static final Logger LOGGER = LoggerFactory.getLogger(FacebookMarketingOAuthFlowIntegrationTest.class); private static final String REDIRECT_URL = "http://localhost/code"; - private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); + private static final Path CREDENTIALS_PATH = Path.of("secrets/facebook_marketing.json"); private ConfigRepository configRepository; - private GoogleOAuthFlow googleOAuthFlow; + private FacebookMarketingOAuthFlow facebookMarketingOAuthFlow; private HttpServer server; private ServerHandler serverHandler; @@ -73,7 +71,7 @@ public void setup() throws IOException { "Must provide path to a oauth credentials file."); } configRepository = mock(ConfigRepository.class); - googleOAuthFlow = new GoogleAnalyticsOauthFlow(configRepository); + facebookMarketingOAuthFlow = new FacebookMarketingOAuthFlow(configRepository); server = HttpServer.create(new InetSocketAddress(80), 0); server.setExecutor(null); // creates a default executor @@ -102,7 +100,7 @@ public void testFullGoogleOAuthFlow() throws InterruptedException, ConfigNotFoun .put("client_id", credentialsJson.get("client_id").asText()) .put("client_secret", credentialsJson.get("client_secret").asText()) .build())))); - final String url = googleOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + final String url = facebookMarketingOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); LOGGER.info("Waiting for user consent at: {}", url); // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing // access... @@ -111,11 +109,11 @@ public void testFullGoogleOAuthFlow() throws InterruptedException, ConfigNotFoun limit -= 1; } assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); - final Map params = googleOAuthFlow.completeSourceOAuth(workspaceId, definitionId, + final Map params = facebookMarketingOAuthFlow.completeSourceOAuth(workspaceId, definitionId, Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); - assertTrue(params.containsKey("refresh_token")); - assertTrue(params.get("refresh_token").toString().length() > 0); + assertTrue(params.containsKey("access_token")); + assertTrue(params.get("access_token").toString().length() > 0); } static class ServerHandler implements HttpHandler { diff --git a/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowIntegrationTest.java new file mode 100644 index 000000000000..d21041b64064 --- /dev/null +++ b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowIntegrationTest.java @@ -0,0 +1,188 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GoogleAdsOAuthFlowIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(GoogleAdsOAuthFlowIntegrationTest.class); + private static final String REDIRECT_URL = "http://localhost/code"; + private static final Path CREDENTIALS_PATH = Path.of("secrets/google_ads.json"); + + private ConfigRepository configRepository; + private GoogleAdsOAuthFlow googleAdsOAuthFlow; + private HttpServer server; + private ServerHandler serverHandler; + + @BeforeEach + public void setup() throws IOException { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a oauth credentials file."); + } + configRepository = mock(ConfigRepository.class); + googleAdsOAuthFlow = new GoogleAdsOAuthFlow(configRepository); + + server = HttpServer.create(new InetSocketAddress(80), 0); + server.setExecutor(null); // creates a default executor + server.start(); + serverHandler = new ServerHandler("code"); + server.createContext("/code", serverHandler); + } + + @AfterEach + void tearDown() { + server.stop(1); + } + + @Test + public void testFullGoogleOAuthFlow() throws InterruptedException, ConfigNotFoundException, IOException, JsonValidationException { + int limit = 20; + final UUID workspaceId = UUID.randomUUID(); + final UUID definitionId = UUID.randomUUID(); + final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() + .put("client_id", credentialsJson.get("credentials").get("client_id").asText()) + .put("client_secret", credentialsJson.get("credentials").get("client_secret").asText()) + .build()))))); + final String url = googleAdsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + LOGGER.info("Waiting for user consent at: {}", url); + // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing + // access... + while (!serverHandler.isSucceeded() && limit > 0) { + Thread.sleep(1000); + limit -= 1; + } + assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); + final Map params = googleAdsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, + Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); + LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); + assertTrue(params.containsKey("credentials")); + final Map credentials = (Map) params.get("credentials"); + assertTrue(credentials.containsKey("refresh_token")); + assertTrue(credentials.get("refresh_token").toString().length() > 0); + assertTrue(credentials.containsKey("access_token")); + assertTrue(credentials.get("access_token").toString().length() > 0); + } + + static class ServerHandler implements HttpHandler { + + final private String expectedParam; + private String paramValue; + private boolean succeeded; + + public ServerHandler(String expectedParam) { + this.expectedParam = expectedParam; + this.paramValue = ""; + this.succeeded = false; + } + + public boolean isSucceeded() { + return succeeded; + } + + public String getParamValue() { + return paramValue; + } + + @Override + public void handle(HttpExchange t) { + final String query = t.getRequestURI().getQuery(); + LOGGER.info("Received query: '{}'", query); + final Map data; + try { + data = deserialize(query); + final String response; + if (data != null && data.containsKey(expectedParam)) { + paramValue = data.get(expectedParam); + response = String.format("Successfully extracted %s:\n'%s'\nTest should be continuing the OAuth Flow to retrieve the refresh_token...", + expectedParam, paramValue); + LOGGER.info(response); + t.sendResponseHeaders(200, response.length()); + succeeded = true; + } else { + response = String.format("Unable to parse query params from redirected url: %s", query); + t.sendResponseHeaders(500, response.length()); + } + final OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } catch (RuntimeException | IOException e) { + LOGGER.error("Failed to parse from body {}", query, e); + } + } + + private static Map deserialize(String query) { + if (query == null) { + return null; + } + final Map result = new HashMap<>(); + for (String param : query.split("&")) { + String[] entry = param.split("="); + if (entry.length > 1) { + result.put(entry[0], entry[1]); + } else { + result.put(entry[0], ""); + } + } + return result; + } + + } + +} diff --git a/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowIntegrationTest.java new file mode 100644 index 000000000000..28db741a0ca1 --- /dev/null +++ b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowIntegrationTest.java @@ -0,0 +1,188 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GoogleAnalyticsOAuthFlowIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(GoogleAnalyticsOAuthFlowIntegrationTest.class); + private static final String REDIRECT_URL = "http://localhost/code"; + private static final Path CREDENTIALS_PATH = Path.of("secrets/google_analytics.json"); + + private ConfigRepository configRepository; + private GoogleAnalyticsOAuthFlow googleAnalyticsOAuthFlow; + private HttpServer server; + private ServerHandler serverHandler; + + @BeforeEach + public void setup() throws IOException { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a oauth credentials file."); + } + configRepository = mock(ConfigRepository.class); + googleAnalyticsOAuthFlow = new GoogleAnalyticsOAuthFlow(configRepository); + + server = HttpServer.create(new InetSocketAddress(80), 0); + server.setExecutor(null); // creates a default executor + server.start(); + serverHandler = new ServerHandler("code"); + server.createContext("/code", serverHandler); + } + + @AfterEach + void tearDown() { + server.stop(1); + } + + @Test + public void testFullGoogleOAuthFlow() throws InterruptedException, ConfigNotFoundException, IOException, JsonValidationException { + int limit = 20; + final UUID workspaceId = UUID.randomUUID(); + final UUID definitionId = UUID.randomUUID(); + final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() + .put("client_id", credentialsJson.get("credentials").get("client_id").asText()) + .put("client_secret", credentialsJson.get("credentials").get("client_secret").asText()) + .build()))))); + final String url = googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + LOGGER.info("Waiting for user consent at: {}", url); + // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing + // access... + while (!serverHandler.isSucceeded() && limit > 0) { + Thread.sleep(1000); + limit -= 1; + } + assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); + final Map params = googleAnalyticsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, + Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); + LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); + assertTrue(params.containsKey("credentials")); + final Map credentials = (Map) params.get("credentials"); + assertTrue(credentials.containsKey("refresh_token")); + assertTrue(credentials.get("refresh_token").toString().length() > 0); + assertTrue(credentials.containsKey("access_token")); + assertTrue(credentials.get("access_token").toString().length() > 0); + } + + static class ServerHandler implements HttpHandler { + + final private String expectedParam; + private String paramValue; + private boolean succeeded; + + public ServerHandler(String expectedParam) { + this.expectedParam = expectedParam; + this.paramValue = ""; + this.succeeded = false; + } + + public boolean isSucceeded() { + return succeeded; + } + + public String getParamValue() { + return paramValue; + } + + @Override + public void handle(HttpExchange t) { + final String query = t.getRequestURI().getQuery(); + LOGGER.info("Received query: '{}'", query); + final Map data; + try { + data = deserialize(query); + final String response; + if (data != null && data.containsKey(expectedParam)) { + paramValue = data.get(expectedParam); + response = String.format("Successfully extracted %s:\n'%s'\nTest should be continuing the OAuth Flow to retrieve the refresh_token...", + expectedParam, paramValue); + LOGGER.info(response); + t.sendResponseHeaders(200, response.length()); + succeeded = true; + } else { + response = String.format("Unable to parse query params from redirected url: %s", query); + t.sendResponseHeaders(500, response.length()); + } + final OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } catch (RuntimeException | IOException e) { + LOGGER.error("Failed to parse from body {}", query, e); + } + } + + private static Map deserialize(String query) { + if (query == null) { + return null; + } + final Map result = new HashMap<>(); + for (String param : query.split("&")) { + String[] entry = param.split("="); + if (entry.length > 1) { + result.put(entry[0], entry[1]); + } else { + result.put(entry[0], ""); + } + } + return result; + } + + } + +} diff --git a/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowIntegrationTest.java b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowIntegrationTest.java new file mode 100644 index 000000000000..c7a2f15d1748 --- /dev/null +++ b/airbyte-oauth/src/test-integration/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowIntegrationTest.java @@ -0,0 +1,188 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GoogleSearchConsoleOAuthFlowIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(GoogleSearchConsoleOAuthFlowIntegrationTest.class); + private static final String REDIRECT_URL = "http://localhost/code"; + private static final Path CREDENTIALS_PATH = Path.of("secrets/google_search_console.json"); + + private ConfigRepository configRepository; + private GoogleSearchConsoleOAuthFlow googleSearchConsoleOAuthFlow; + private HttpServer server; + private ServerHandler serverHandler; + + @BeforeEach + public void setup() throws IOException { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a oauth credentials file."); + } + configRepository = mock(ConfigRepository.class); + googleSearchConsoleOAuthFlow = new GoogleSearchConsoleOAuthFlow(configRepository); + + server = HttpServer.create(new InetSocketAddress(80), 0); + server.setExecutor(null); // creates a default executor + server.start(); + serverHandler = new ServerHandler("code"); + server.createContext("/code", serverHandler); + } + + @AfterEach + void tearDown() { + server.stop(1); + } + + @Test + public void testFullGoogleOAuthFlow() throws InterruptedException, ConfigNotFoundException, IOException, JsonValidationException { + int limit = 20; + final UUID workspaceId = UUID.randomUUID(); + final UUID definitionId = UUID.randomUUID(); + final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); + final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(Map.of("authorization", ImmutableMap.builder() + .put("client_id", credentialsJson.get("authorization").get("client_id").asText()) + .put("client_secret", credentialsJson.get("authorization").get("client_secret").asText()) + .build()))))); + final String url = googleSearchConsoleOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + LOGGER.info("Waiting for user consent at: {}", url); + // TODO: To automate, start a selenium job to navigate to the Consent URL and click on allowing + // access... + while (!serverHandler.isSucceeded() && limit > 0) { + Thread.sleep(1000); + limit -= 1; + } + assertTrue(serverHandler.isSucceeded(), "Failed to get User consent on time"); + final Map params = googleSearchConsoleOAuthFlow.completeSourceOAuth(workspaceId, definitionId, + Map.of("code", serverHandler.getParamValue()), REDIRECT_URL); + LOGGER.info("Response from completing OAuth Flow is: {}", params.toString()); + assertTrue(params.containsKey("authorization")); + final Map credentials = (Map) params.get("authorization"); + assertTrue(credentials.containsKey("refresh_token")); + assertTrue(credentials.get("refresh_token").toString().length() > 0); + assertTrue(credentials.containsKey("access_token")); + assertTrue(credentials.get("access_token").toString().length() > 0); + } + + static class ServerHandler implements HttpHandler { + + final private String expectedParam; + private String paramValue; + private boolean succeeded; + + public ServerHandler(String expectedParam) { + this.expectedParam = expectedParam; + this.paramValue = ""; + this.succeeded = false; + } + + public boolean isSucceeded() { + return succeeded; + } + + public String getParamValue() { + return paramValue; + } + + @Override + public void handle(HttpExchange t) { + final String query = t.getRequestURI().getQuery(); + LOGGER.info("Received query: '{}'", query); + final Map data; + try { + data = deserialize(query); + final String response; + if (data != null && data.containsKey(expectedParam)) { + paramValue = data.get(expectedParam); + response = String.format("Successfully extracted %s:\n'%s'\nTest should be continuing the OAuth Flow to retrieve the refresh_token...", + expectedParam, paramValue); + LOGGER.info(response); + t.sendResponseHeaders(200, response.length()); + succeeded = true; + } else { + response = String.format("Unable to parse query params from redirected url: %s", query); + t.sendResponseHeaders(500, response.length()); + } + final OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } catch (RuntimeException | IOException e) { + LOGGER.error("Failed to parse from body {}", query, e); + } + } + + private static Map deserialize(String query) { + if (query == null) { + return null; + } + final Map result = new HashMap<>(); + for (String param : query.split("&")) { + String[] entry = param.split("="); + if (entry.length > 1) { + result.put(entry[0], entry[1]); + } else { + result.put(entry[0], ""); + } + } + return result; + } + + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlowTest.java new file mode 100644 index 000000000000..5bb0854b8ee4 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/FacebookMarketingOAuthFlowTest.java @@ -0,0 +1,117 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class FacebookMarketingOAuthFlowTest { + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private HttpClient httpClient; + private ConfigRepository configRepository; + private FacebookMarketingOAuthFlow facebookMarketingOAuthFlow; + + private UUID workspaceId; + private UUID definitionId; + + @BeforeEach + public void setup() { + httpClient = mock(HttpClient.class); + configRepository = mock(ConfigRepository.class); + facebookMarketingOAuthFlow = new FacebookMarketingOAuthFlow(configRepository, httpClient, FacebookMarketingOAuthFlowTest::getConstantState); + + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + } + + private static String getConstantState() { + return "state"; + } + + @Test + public void testCompleteSourceOAuth() throws IOException, JsonValidationException, InterruptedException, ConfigNotFoundException { + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build())))); + + Map returnedCredentials = Map.of("access_token", "access_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = + facebookMarketingOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + + assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + } + + @Test + public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { + when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withDestinationDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build())))); + + Map returnedCredentials = Map.of("access_token", "access_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = + facebookMarketingOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + + assertEquals(Jsons.serialize(returnedCredentials), Jsons.serialize(actualQueryParams)); + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/google/GoogleAdsOauthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java similarity index 88% rename from airbyte-oauth/src/test/java/io/airbyte/oauth/google/GoogleAdsOauthFlowTest.java rename to airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java index 45807d7d28e8..db82e4f3b878 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/google/GoogleAdsOauthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAdsOAuthFlowTest.java @@ -22,7 +22,7 @@ * SOFTWARE. */ -package io.airbyte.oauth.google; +package io.airbyte.oauth.flows.google; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -46,14 +46,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -public class GoogleAdsOauthFlowTest { +public class GoogleAdsOAuthFlowTest { - private static final String SCOPE = "https%3A//www.googleapis.com/auth/analytics.readonly"; private static final String REDIRECT_URL = "https://airbyte.io"; private HttpClient httpClient; private ConfigRepository configRepository; - private GoogleAdsOauthFlow googleAdsOauthFlow; + private GoogleAdsOAuthFlow googleAdsOAuthFlow; private UUID workspaceId; private UUID definitionId; @@ -62,12 +61,16 @@ public class GoogleAdsOauthFlowTest { public void setup() { httpClient = mock(HttpClient.class); configRepository = mock(ConfigRepository.class); - googleAdsOauthFlow = new GoogleAdsOauthFlow(configRepository, httpClient); + googleAdsOAuthFlow = new GoogleAdsOAuthFlow(configRepository, httpClient, GoogleAdsOAuthFlowTest::getConstantState); workspaceId = UUID.randomUUID(); definitionId = UUID.randomUUID(); } + private static String getConstantState() { + return "state"; + } + @Test public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() @@ -84,7 +87,7 @@ public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundExceptio when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); when(httpClient.send(any(), any())).thenReturn(response); final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleAdsOauthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + final Map actualQueryParams = googleAdsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); } @@ -105,7 +108,7 @@ public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundExc when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); when(httpClient.send(any(), any())).thenReturn(response); final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleAdsOauthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + final Map actualQueryParams = googleAdsOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); } @@ -116,8 +119,8 @@ public void testGetClientIdUnsafe() { Map clientIdMap = Map.of("client_id", clientId); Map> nestedConfig = Map.of("credentials", clientIdMap); - assertThrows(IllegalArgumentException.class, () -> googleAdsOauthFlow.getClientIdUnsafe(Jsons.jsonNode(clientIdMap))); - assertEquals(clientId, googleAdsOauthFlow.getClientIdUnsafe(Jsons.jsonNode(nestedConfig))); + assertThrows(IllegalArgumentException.class, () -> googleAdsOAuthFlow.getClientIdUnsafe(Jsons.jsonNode(clientIdMap))); + assertEquals(clientId, googleAdsOAuthFlow.getClientIdUnsafe(Jsons.jsonNode(nestedConfig))); } @Test @@ -126,8 +129,8 @@ public void testGetClientSecretUnsafe() { Map clientIdMap = Map.of("client_secret", clientSecret); Map> nestedConfig = Map.of("credentials", clientIdMap); - assertThrows(IllegalArgumentException.class, () -> googleAdsOauthFlow.getClientSecretUnsafe(Jsons.jsonNode(clientIdMap))); - assertEquals(clientSecret, googleAdsOauthFlow.getClientSecretUnsafe(Jsons.jsonNode(nestedConfig))); + assertThrows(IllegalArgumentException.class, () -> googleAdsOAuthFlow.getClientSecretUnsafe(Jsons.jsonNode(clientIdMap))); + assertEquals(clientSecret, googleAdsOAuthFlow.getClientSecretUnsafe(Jsons.jsonNode(nestedConfig))); } } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/google/GoogleOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java similarity index 65% rename from airbyte-oauth/src/test/java/io/airbyte/oauth/google/GoogleOAuthFlowTest.java rename to airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java index c9992dadb556..0fc09e044b7c 100644 --- a/airbyte-oauth/src/test/java/io/airbyte/oauth/google/GoogleOAuthFlowTest.java +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleAnalyticsOAuthFlowTest.java @@ -22,7 +22,7 @@ * SOFTWARE. */ -package io.airbyte.oauth.google; +package io.airbyte.oauth.flows.google; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -39,10 +39,8 @@ import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; -import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -53,16 +51,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class GoogleOAuthFlowTest { +public class GoogleAnalyticsOAuthFlowTest { - private static final Logger LOGGER = LoggerFactory.getLogger(GoogleOAuthFlowTest.class); + private static final Logger LOGGER = LoggerFactory.getLogger(GoogleAnalyticsOAuthFlowTest.class); private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); private static final String REDIRECT_URL = "https://airbyte.io"; - private static final String SCOPE = "https://www.googleapis.com/auth/analytics.readonly"; + private static final String EXPECTED_REDIRECT_URL = "https%3A%2F%2Fairbyte.io"; + private static final String EXPECTED_SCOPE = "https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fanalytics.readonly"; private HttpClient httpClient; private ConfigRepository configRepository; - private GoogleOAuthFlow googleOAuthFlow; + private GoogleAnalyticsOAuthFlow googleAnalyticsOAuthFlow; private UUID workspaceId; private UUID definitionId; @@ -71,16 +70,20 @@ public class GoogleOAuthFlowTest { public void setup() { httpClient = mock(HttpClient.class); configRepository = mock(ConfigRepository.class); - googleOAuthFlow = new GoogleOAuthFlow(configRepository, SCOPE, httpClient); + googleAnalyticsOAuthFlow = new GoogleAnalyticsOAuthFlow(configRepository, httpClient, GoogleAnalyticsOAuthFlowTest::getConstantState); workspaceId = UUID.randomUUID(); definitionId = UUID.randomUUID(); } + private static String getConstantState() { + return "state"; + } + @Test public void testGetConsentUrlEmptyOAuthParameters() { - assertThrows(ConfigNotFoundException.class, () -> googleOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); - assertThrows(ConfigNotFoundException.class, () -> googleOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + assertThrows(ConfigNotFoundException.class, () -> googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + assertThrows(ConfigNotFoundException.class, () -> googleAnalyticsOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); } @Test @@ -95,8 +98,8 @@ public void testGetConsentUrlIncompleteOAuthParameters() throws IOException, Jso .withDestinationDefinitionId(definitionId) .withWorkspaceId(workspaceId) .withConfiguration(Jsons.emptyObject()))); - assertThrows(IllegalArgumentException.class, () -> googleOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); - assertThrows(IllegalArgumentException.class, () -> googleOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + assertThrows(IllegalArgumentException.class, () -> googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL)); + assertThrows(IllegalArgumentException.class, () -> googleAnalyticsOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL)); } @Test @@ -105,16 +108,16 @@ public void testGetSourceConsentUrl() throws IOException, ConfigNotFoundExceptio .withOauthParameterId(UUID.randomUUID()) .withSourceDefinitionId(definitionId) .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() .put("client_id", getClientId()) - .build())))); - final String actualSourceUrl = googleOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); + .build()))))); + final String actualSourceUrl = googleAnalyticsOAuthFlow.getSourceConsentUrl(workspaceId, definitionId, REDIRECT_URL); final String expectedSourceUrl = String.format( - "https://accounts.google.com/o/oauth2/v2/auth?state=%s&client_id=%s&redirect_uri=%s&scope=%s&access_type=offline&include_granted_scopes=true&response_type=code&prompt=consent", - definitionId, + "https://accounts.google.com/o/oauth2/v2/auth?client_id=%s&redirect_uri=%s&response_type=code&scope=%s&access_type=offline&state=%s&include_granted_scopes=true&prompt=consent", getClientId(), - urlEncode(REDIRECT_URL), - urlEncode(SCOPE)); + EXPECTED_REDIRECT_URL, + EXPECTED_SCOPE, + getConstantState()); LOGGER.info(expectedSourceUrl); assertEquals(expectedSourceUrl, actualSourceUrl); } @@ -125,19 +128,19 @@ public void testGetDestinationConsentUrl() throws IOException, ConfigNotFoundExc .withOauthParameterId(UUID.randomUUID()) .withDestinationDefinitionId(definitionId) .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() .put("client_id", getClientId()) - .build())))); + .build()))))); // It would be better to make this comparison agnostic of the order of query params but the URI // class' equals() method // considers URLs with different qparam orders different URIs.. - final String actualDestinationUrl = googleOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL); + final String actualDestinationUrl = googleAnalyticsOAuthFlow.getDestinationConsentUrl(workspaceId, definitionId, REDIRECT_URL); final String expectedDestinationUrl = String.format( - "https://accounts.google.com/o/oauth2/v2/auth?state=%s&client_id=%s&redirect_uri=%s&scope=%s&access_type=offline&include_granted_scopes=true&response_type=code&prompt=consent", - definitionId, + "https://accounts.google.com/o/oauth2/v2/auth?client_id=%s&redirect_uri=%s&response_type=code&scope=%s&access_type=offline&state=%s&include_granted_scopes=true&prompt=consent", getClientId(), - urlEncode(REDIRECT_URL), - urlEncode(SCOPE)); + EXPECTED_REDIRECT_URL, + EXPECTED_SCOPE, + getConstantState()); LOGGER.info(expectedDestinationUrl); assertEquals(expectedDestinationUrl, actualDestinationUrl); } @@ -148,12 +151,12 @@ public void testCompleteOAuthMissingCode() throws IOException, ConfigNotFoundExc .withOauthParameterId(UUID.randomUUID()) .withSourceDefinitionId(definitionId) .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() .put("client_id", getClientId()) .put("client_secret", "test_client_secret") - .build())))); + .build()))))); final Map queryParams = Map.of(); - assertThrows(IOException.class, () -> googleOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL)); + assertThrows(IOException.class, () -> googleAnalyticsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL)); } @Test @@ -162,18 +165,17 @@ public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundExceptio .withOauthParameterId(UUID.randomUUID()) .withSourceDefinitionId(definitionId) .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() .put("client_id", getClientId()) .put("client_secret", "test_client_secret") - .build())))); - final String expectedQueryParams = Jsons.serialize(Map.of( - "refresh_token", "refresh_token_response")); + .build()))))); + Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(expectedQueryParams); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); when(httpClient.send(any(), any())).thenReturn(response); final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(expectedQueryParams, Jsons.serialize(actualQueryParams)); + final Map actualQueryParams = googleAnalyticsOAuthFlow.completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); } @Test @@ -182,18 +184,18 @@ public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundExc .withOauthParameterId(UUID.randomUUID()) .withDestinationDefinitionId(definitionId) .withWorkspaceId(workspaceId) - .withConfiguration(Jsons.jsonNode(ImmutableMap.builder() + .withConfiguration(Jsons.jsonNode(Map.of("credentials", ImmutableMap.builder() .put("client_id", getClientId()) .put("client_secret", "test_client_secret") - .build())))); - final String expectedQueryParams = Jsons.serialize(Map.of( - "refresh_token", "refresh_token_response")); + .build()))))); + Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); final HttpResponse response = mock(HttpResponse.class); - when(response.body()).thenReturn(expectedQueryParams); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); when(httpClient.send(any(), any())).thenReturn(response); final Map queryParams = Map.of("code", "test_code"); - final Map actualQueryParams = googleOAuthFlow.completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); - assertEquals(expectedQueryParams, Jsons.serialize(actualQueryParams)); + final Map actualQueryParams = googleAnalyticsOAuthFlow + .completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + assertEquals(Jsons.serialize(Map.of("credentials", returnedCredentials)), Jsons.serialize(actualQueryParams)); } private String getClientId() throws IOException { @@ -202,12 +204,8 @@ private String getClientId() throws IOException { } else { final String fullConfigAsString = new String(Files.readAllBytes(CREDENTIALS_PATH)); final JsonNode credentialsJson = Jsons.deserialize(fullConfigAsString); - return credentialsJson.get("client_id").asText(); + return credentialsJson.get("credentials").get("client_id").asText(); } } - private String urlEncode(String s) { - return URLEncoder.encode(s, StandardCharsets.UTF_8); - } - } diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java new file mode 100644 index 000000000000..faeda5df38cd --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/google/GoogleSearchConsoleOAuthFlowTest.java @@ -0,0 +1,138 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.oauth.flows.google; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.DestinationOAuthParameter; +import io.airbyte.config.SourceOAuthParameter; +import io.airbyte.config.persistence.ConfigNotFoundException; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.validation.json.JsonValidationException; +import java.io.IOException; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class GoogleSearchConsoleOAuthFlowTest { + + private static final String REDIRECT_URL = "https://airbyte.io"; + + private HttpClient httpClient; + private ConfigRepository configRepository; + private GoogleSearchConsoleOAuthFlow googleSearchConsoleOAuthFlow; + + private UUID workspaceId; + private UUID definitionId; + + @BeforeEach + public void setup() { + httpClient = mock(HttpClient.class); + configRepository = mock(ConfigRepository.class); + googleSearchConsoleOAuthFlow = new GoogleSearchConsoleOAuthFlow(configRepository, httpClient, GoogleSearchConsoleOAuthFlowTest::getConstantState); + + workspaceId = UUID.randomUUID(); + definitionId = UUID.randomUUID(); + } + + private static String getConstantState() { + return "state"; + } + + @Test + public void testCompleteSourceOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { + when(configRepository.listSourceOAuthParam()).thenReturn(List.of(new SourceOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withSourceDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(Map.of("authorization", ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build()))))); + + Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = googleSearchConsoleOAuthFlow + .completeSourceOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + + assertEquals(Jsons.serialize(Map.of("authorization", returnedCredentials)), Jsons.serialize(actualQueryParams)); + } + + @Test + public void testCompleteDestinationOAuth() throws IOException, ConfigNotFoundException, JsonValidationException, InterruptedException { + when(configRepository.listDestinationOAuthParam()).thenReturn(List.of(new DestinationOAuthParameter() + .withOauthParameterId(UUID.randomUUID()) + .withDestinationDefinitionId(definitionId) + .withWorkspaceId(workspaceId) + .withConfiguration(Jsons.jsonNode(Map.of("authorization", ImmutableMap.builder() + .put("client_id", "test_client_id") + .put("client_secret", "test_client_secret") + .build()))))); + + Map returnedCredentials = Map.of("refresh_token", "refresh_token_response"); + final HttpResponse response = mock(HttpResponse.class); + when(response.body()).thenReturn(Jsons.serialize(returnedCredentials)); + when(httpClient.send(any(), any())).thenReturn(response); + final Map queryParams = Map.of("code", "test_code"); + final Map actualQueryParams = googleSearchConsoleOAuthFlow + .completeDestinationOAuth(workspaceId, definitionId, queryParams, REDIRECT_URL); + + assertEquals(Jsons.serialize(Map.of("authorization", returnedCredentials)), Jsons.serialize(actualQueryParams)); + } + + @Test + public void testGetClientIdUnsafe() { + String clientId = "123"; + Map clientIdMap = Map.of("client_id", clientId); + Map> nestedConfig = Map.of("authorization", clientIdMap); + + assertThrows(IllegalArgumentException.class, () -> googleSearchConsoleOAuthFlow.getClientIdUnsafe(Jsons.jsonNode(clientIdMap))); + assertEquals(clientId, googleSearchConsoleOAuthFlow.getClientIdUnsafe(Jsons.jsonNode(nestedConfig))); + } + + @Test + public void testGetClientSecretUnsafe() { + String clientSecret = "secret"; + Map clientIdMap = Map.of("client_secret", clientSecret); + Map> nestedConfig = Map.of("authorization", clientIdMap); + + assertThrows(IllegalArgumentException.class, () -> googleSearchConsoleOAuthFlow.getClientSecretUnsafe(Jsons.jsonNode(clientIdMap))); + assertEquals(clientSecret, googleSearchConsoleOAuthFlow.getClientSecretUnsafe(Jsons.jsonNode(nestedConfig))); + } + +} diff --git a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/JobScheduler.java b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/JobScheduler.java index 0397007bdf9e..41196eb2da29 100644 --- a/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/JobScheduler.java +++ b/airbyte-scheduler/app/src/main/java/io/airbyte/scheduler/app/JobScheduler.java @@ -40,7 +40,6 @@ import java.time.Instant; import java.util.List; import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiPredicate; import java.util.stream.Collectors; import org.slf4j.Logger; @@ -92,7 +91,7 @@ public void run() { } private void scheduleSyncJobs() throws IOException { - final AtomicInteger jobsScheduled = new AtomicInteger(); + int jobsScheduled = 0; final List activeConnections = getAllActiveConnections(); for (StandardSync connection : activeConnections) { @@ -100,11 +99,12 @@ private void scheduleSyncJobs() throws IOException { if (scheduleJobPredicate.test(previousJobOptional, connection)) { jobFactory.create(connection.getConnectionId()); + jobsScheduled++; } } - int jobsScheduledCount = jobsScheduled.get(); - if (jobsScheduledCount > 0) { - LOGGER.info("Job-Scheduler Summary. Active connections: {}, Jobs scheduler: {}", activeConnections.size(), jobsScheduled.get()); + + if (jobsScheduled > 0) { + LOGGER.info("Job-Scheduler Summary. Active connections: {}, Jobs scheduled this cycle: {}", activeConnections.size(), jobsScheduled); } } diff --git a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/WorkspaceHelperTest.java b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/WorkspaceHelperTest.java index b6fd60f3df6b..67bdbcc43877 100644 --- a/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/WorkspaceHelperTest.java +++ b/airbyte-scheduler/persistence/src/test/java/io/airbyte/scheduler/persistence/WorkspaceHelperTest.java @@ -149,13 +149,13 @@ public void testSource() throws IOException, JsonValidationException { @Test public void testDestination() throws IOException, JsonValidationException { configRepository.writeStandardDestinationDefinition(DEST_DEF); - configRepository.writeDestinationConnection(DEST); + configRepository.writeDestinationConnection(DEST, emptyConnectorSpec); final UUID retrievedWorkspace = workspaceHelper.getWorkspaceForDestinationIdIgnoreExceptions(DEST_ID); assertEquals(WORKSPACE_ID, retrievedWorkspace); // check that caching is working - configRepository.writeDestinationConnection(Jsons.clone(DEST).withWorkspaceId(UUID.randomUUID())); + configRepository.writeDestinationConnection(Jsons.clone(DEST).withWorkspaceId(UUID.randomUUID()), emptyConnectorSpec); final UUID retrievedWorkspaceAfterUpdate = workspaceHelper.getWorkspaceForDestinationIdIgnoreExceptions(DEST_ID); assertEquals(WORKSPACE_ID, retrievedWorkspaceAfterUpdate); } @@ -165,7 +165,7 @@ public void testConnection() throws IOException, JsonValidationException { configRepository.writeStandardSource(SOURCE_DEF); configRepository.writeSourceConnection(SOURCE, emptyConnectorSpec); configRepository.writeStandardDestinationDefinition(DEST_DEF); - configRepository.writeDestinationConnection(DEST); + configRepository.writeDestinationConnection(DEST, emptyConnectorSpec); // set up connection configRepository.writeStandardSync(CONNECTION); @@ -181,7 +181,7 @@ public void testConnection() throws IOException, JsonValidationException { // check that caching is working final UUID newWorkspace = UUID.randomUUID(); configRepository.writeSourceConnection(Jsons.clone(SOURCE).withWorkspaceId(newWorkspace), emptyConnectorSpec); - configRepository.writeDestinationConnection(Jsons.clone(DEST).withWorkspaceId(newWorkspace)); + configRepository.writeDestinationConnection(Jsons.clone(DEST).withWorkspaceId(newWorkspace), emptyConnectorSpec); final UUID retrievedWorkspaceAfterUpdate = workspaceHelper.getWorkspaceForDestinationIdIgnoreExceptions(DEST_ID); assertEquals(WORKSPACE_ID, retrievedWorkspaceAfterUpdate); } @@ -205,7 +205,7 @@ public void testConnectionAndJobs() throws IOException, JsonValidationException configRepository.writeStandardSource(SOURCE_DEF); configRepository.writeSourceConnection(SOURCE, emptyConnectorSpec); configRepository.writeStandardDestinationDefinition(DEST_DEF); - configRepository.writeDestinationConnection(DEST); + configRepository.writeDestinationConnection(DEST, emptyConnectorSpec); configRepository.writeStandardSync(CONNECTION); // test jobs diff --git a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java index a30165a1ae10..bb483d303678 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ConfigDumpImporter.java @@ -49,12 +49,12 @@ import io.airbyte.config.persistence.ConfigPersistence; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.db.instance.jobs.JobsDatabaseSchema; -import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.persistence.DefaultJobPersistence; import io.airbyte.scheduler.persistence.JobPersistence; import io.airbyte.scheduler.persistence.WorkspaceHelper; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.server.errors.IdNotFoundKnownException; +import io.airbyte.server.handlers.DestinationHandler; import io.airbyte.server.handlers.SourceHandler; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; @@ -89,14 +89,13 @@ public class ConfigDumpImporter { private static final String CONFIG_FOLDER_NAME = "airbyte_config"; private static final String DB_FOLDER_NAME = "airbyte_db"; private static final String VERSION_FILE_NAME = "VERSION"; - private static final String TMP_AIRBYTE_STAGED_RESOURCES = "/tmp/airbyte_staged_resources"; + private static final Path TMP_AIRBYTE_STAGED_RESOURCES = Path.of("/tmp/airbyte_staged_resources"); private final ConfigRepository configRepository; private final WorkspaceHelper workspaceHelper; private final SpecFetcher specFetcher; private final JsonSchemaValidator jsonSchemaValidator; private final JobPersistence jobPersistence; - private final Path stagedResourceRoot; public ConfigDumpImporter(ConfigRepository configRepository, JobPersistence jobPersistence, @@ -116,13 +115,22 @@ public ConfigDumpImporter(ConfigRepository configRepository, this.configRepository = configRepository; this.workspaceHelper = workspaceHelper; this.specFetcher = specFetcher; + } + + /** + * Re-initialize the staged resource folder that contains uploaded artifacts when importing + * workspaces. This is because they need to be done in two steps (two API endpoints), upload + * resource first then import. When server starts, we flush the content of this folder, deleting + * previously staged resources that were not imported yet. + */ + public static void initStagedResourceFolder() { try { - this.stagedResourceRoot = Path.of(TMP_AIRBYTE_STAGED_RESOURCES); - if (stagedResourceRoot.toFile().exists()) { - FileUtils.forceDelete(stagedResourceRoot.toFile()); + File stagedResourceRoot = TMP_AIRBYTE_STAGED_RESOURCES.toFile(); + if (stagedResourceRoot.exists()) { + FileUtils.forceDelete(stagedResourceRoot); } - FileUtils.forceMkdir(stagedResourceRoot.toFile()); - FileUtils.forceDeleteOnExit(stagedResourceRoot.toFile()); + FileUtils.forceMkdir(stagedResourceRoot); + FileUtils.forceDeleteOnExit(stagedResourceRoot); } catch (IOException e) { throw new RuntimeException("Failed to create staging resource folder", e); } @@ -325,7 +333,7 @@ private void checkDBVersion(final String airbyteVersion) throws IOException { public UploadRead uploadArchiveResource(File archive) { try { final UUID resourceId = UUID.randomUUID(); - FileUtils.moveFile(archive, stagedResourceRoot.resolve(resourceId.toString()).toFile()); + FileUtils.moveFile(archive, TMP_AIRBYTE_STAGED_RESOURCES.resolve(resourceId.toString()).toFile()); return new UploadRead() .status(UploadRead.StatusEnum.SUCCEEDED) .resourceId(resourceId); @@ -336,7 +344,7 @@ public UploadRead uploadArchiveResource(File archive) { } public File getArchiveResource(UUID resourceId) { - final File archive = stagedResourceRoot.resolve(resourceId.toString()).toFile(); + final File archive = TMP_AIRBYTE_STAGED_RESOURCES.resolve(resourceId.toString()).toFile(); if (!archive.exists()) { throw new IdNotFoundKnownException("Archive Resource not found", resourceId.toString()); } @@ -414,7 +422,6 @@ private void importConfigsIntoWorkspace(Path sourceRoot, UUID workspaceId, b return sourceConnection; }, (sourceConnection) -> { - final ConnectorSpecification spec; // make sure connector definition exists try { final StandardSourceDefinition sourceDefinition = @@ -422,11 +429,10 @@ private void importConfigsIntoWorkspace(Path sourceRoot, UUID workspaceId, b if (sourceDefinition == null) { return; } - spec = SourceHandler.getSpecFromSourceDefinitionId(specFetcher, sourceDefinition); + configRepository.writeSourceConnection(sourceConnection, SourceHandler.getSpecFromSourceDefinitionId(specFetcher, sourceDefinition)); } catch (ConfigNotFoundException e) { return; } - configRepository.writeSourceConnection(sourceConnection, spec); })); case STANDARD_DESTINATION_DEFINITION -> importDestinationDefinitionIntoWorkspace(configs); case DESTINATION_CONNECTION -> destinationIdMap.putAll(importIntoWorkspace( @@ -442,13 +448,15 @@ private void importConfigsIntoWorkspace(Path sourceRoot, UUID workspaceId, b (destinationConnection) -> { // make sure connector definition exists try { - if (configRepository.getStandardDestinationDefinition(destinationConnection.getDestinationDefinitionId()) == null) { + StandardDestinationDefinition destinationDefinition = configRepository.getStandardDestinationDefinition( + destinationConnection.getDestinationDefinitionId()); + if (destinationDefinition == null) { return; } + configRepository.writeDestinationConnection(destinationConnection, DestinationHandler.getSpec(specFetcher, destinationDefinition)); } catch (ConfigNotFoundException e) { return; } - configRepository.writeDestinationConnection(destinationConnection); })); case STANDARD_SYNC -> standardSyncs = configs; case STANDARD_SYNC_OPERATION -> operationIdMap.putAll(importIntoWorkspace( diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 997e87210365..0e9d52917d84 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -34,7 +34,6 @@ import io.airbyte.config.StandardWorkspace; import io.airbyte.config.helpers.LogClientSingleton; import io.airbyte.config.persistence.ConfigRepository; -import io.airbyte.config.persistence.ConfigSeedProvider; import io.airbyte.config.persistence.DatabaseConfigPersistence; import io.airbyte.config.persistence.YamlSeedConfigPersistence; import io.airbyte.db.Database; @@ -175,6 +174,9 @@ public static ServerRunnable getServer(final ServerFactory apiFactory) throws Ex LogClientSingleton.setWorkspaceMdc(LogClientSingleton.getServerLogsRoot(configs)); + LOGGER.info("Creating Staged Resource folder..."); + ConfigDumpImporter.initStagedResourceFolder(); + LOGGER.info("Creating config repository..."); final Database configDatabase = new ConfigsDatabaseInstance( configs.getConfigDatabaseUser(), @@ -182,7 +184,7 @@ public static ServerRunnable getServer(final ServerFactory apiFactory) throws Ex configs.getConfigDatabaseUrl()) .getAndInitialize(); final DatabaseConfigPersistence configPersistence = new DatabaseConfigPersistence(configDatabase); - configPersistence.loadData(ConfigSeedProvider.get(configs)); + configPersistence.migrateFileConfigs(configs); final ConfigRepository configRepository = new ConfigRepository(configPersistence.withValidation()); LOGGER.info("Creating Scheduler persistence..."); @@ -239,11 +241,12 @@ public static ServerRunnable getServer(final ServerFactory apiFactory) throws Ex } } - runFlywayMigration(configs, configDatabase, jobDatabase); - if (airbyteDatabaseVersion.isPresent() && AirbyteVersion.isCompatible(airbyteVersion, airbyteDatabaseVersion.get())) { LOGGER.info("Starting server..."); + runFlywayMigration(configs, configDatabase, jobDatabase); + configPersistence.loadData(YamlSeedConfigPersistence.get()); + return apiFactory.create( schedulerJobClient, cachingSchedulerClient, diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 295b7a59e520..25836806c097 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -89,6 +89,7 @@ import io.airbyte.api.model.WebBackendConnectionRead; import io.airbyte.api.model.WebBackendConnectionReadList; import io.airbyte.api.model.WebBackendConnectionRequestBody; +import io.airbyte.api.model.WebBackendConnectionSearch; import io.airbyte.api.model.WebBackendConnectionUpdate; import io.airbyte.api.model.WorkspaceCreate; import io.airbyte.api.model.WorkspaceIdRequestBody; @@ -585,6 +586,11 @@ public WebBackendConnectionReadList webBackendListConnectionsForWorkspace(final return execute(() -> webBackendConnectionsHandler.webBackendListConnectionsForWorkspace(workspaceIdRequestBody)); } + @Override + public WebBackendConnectionReadList webBackendConnectionSearch(final WebBackendConnectionSearch webBackendConnectionSearch) { + return execute(() -> webBackendConnectionsHandler.webBackendSearchConnections(webBackendConnectionSearch)); + } + @Override public WebBackendConnectionRead webBackendGetConnection(final WebBackendConnectionRequestBody webBackendConnectionRequestBody) { return execute(() -> webBackendConnectionsHandler.webBackendGetConnection(webBackendConnectionRequestBody)); diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java new file mode 100644 index 000000000000..b86ee4e1f45f --- /dev/null +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/OauthModelConverter.java @@ -0,0 +1,50 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.server.converters; + +import io.airbyte.api.model.AuthSpecification; +import io.airbyte.api.model.OAuth2Specification; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.util.Optional; + +public class OauthModelConverter { + + public static Optional getAuthSpec(ConnectorSpecification spec) { + if (spec.getAuthSpecification() == null) { + return Optional.empty(); + } + io.airbyte.protocol.models.AuthSpecification incomingAuthSpec = spec.getAuthSpecification(); + + AuthSpecification authSpecification = new AuthSpecification(); + if (incomingAuthSpec.getAuthType() == io.airbyte.protocol.models.AuthSpecification.AuthType.OAUTH_2_0) { + authSpecification.authType(AuthSpecification.AuthTypeEnum.OAUTH2_0) + .oauth2Specification(new OAuth2Specification() + .oauthFlowInitParameters(incomingAuthSpec.getOauth2Specification().getOauthFlowInitParameters())); + } + + return Optional.ofNullable(authSpecification); + } + +} diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index 1be18b97cd00..98917f4479c6 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -255,6 +255,19 @@ public ConnectionReadList listConnectionsForWorkspace(WorkspaceIdRequestBody wor return new ConnectionReadList().connections(connectionReads); } + public ConnectionReadList listConnections() throws JsonValidationException, ConfigNotFoundException, IOException { + final List connectionReads = Lists.newArrayList(); + + for (StandardSync standardSync : configRepository.listStandardSyncs()) { + if (standardSync.getStatus() == StandardSync.Status.DEPRECATED) { + continue; + } + connectionReads.add(buildConnectionRead(standardSync.getConnectionId())); + } + + return new ConnectionReadList().connections(connectionReads); + } + public ConnectionRead getConnection(ConnectionIdRequestBody connectionIdRequestBody) throws JsonValidationException, IOException, ConfigNotFoundException { return buildConnectionRead(connectionIdRequestBody.getConnectionId()); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java index 8b1dad257be2..e2a1cf73261b 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/DestinationHandler.java @@ -205,11 +205,14 @@ private void validateDestination(final ConnectorSpecification spec, final JsonNo validator.ensure(spec.getConnectionSpecification(), configuration); } - private ConnectorSpecification getSpec(UUID destinationDefinitionId) + public ConnectorSpecification getSpec(UUID destinationDefinitionId) throws JsonValidationException, IOException, ConfigNotFoundException { - final StandardDestinationDefinition destinationDef = configRepository.getStandardDestinationDefinition(destinationDefinitionId); - final String imageName = DockerUtils.getTaggedImageName(destinationDef.getDockerRepository(), destinationDef.getDockerImageTag()); - return specFetcher.execute(imageName); + return getSpec(specFetcher, configRepository.getStandardDestinationDefinition(destinationDefinitionId)); + } + + public static ConnectorSpecification getSpec(SpecFetcher specFetcher, StandardDestinationDefinition destinationDef) + throws JsonValidationException, IOException, ConfigNotFoundException { + return specFetcher.execute(DockerUtils.getTaggedImageName(destinationDef.getDockerRepository(), destinationDef.getDockerImageTag())); } private void persistDestinationConnection(final String name, @@ -218,7 +221,7 @@ private void persistDestinationConnection(final String name, final UUID destinationId, final JsonNode configurationJson, final boolean tombstone) - throws JsonValidationException, IOException { + throws JsonValidationException, IOException, ConfigNotFoundException { final DestinationConnection destinationConnection = new DestinationConnection() .withName(name) .withDestinationDefinitionId(destinationDefinitionId) @@ -226,8 +229,7 @@ private void persistDestinationConnection(final String name, .withDestinationId(destinationId) .withConfiguration(configurationJson) .withTombstone(tombstone); - - configRepository.writeDestinationConnection(destinationConnection); + configRepository.writeDestinationConnection(destinationConnection, getSpec(destinationDefinitionId)); } private DestinationRead buildDestinationRead(final UUID destinationId) throws JsonValidationException, IOException, ConfigNotFoundException { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java index c8e3bb155e18..28d97d04659c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OAuthHandler.java @@ -95,23 +95,25 @@ public Map completeDestinationOAuth(CompleteDestinationOAuthRequ oauthDestinationRequestBody.getRedirectUrl()); } + public void setSourceInstancewideOauthParams(SetInstancewideSourceOauthParamsRequestBody requestBody) throws JsonValidationException, IOException { + final SourceOAuthParameter param = configRepository + .getSourceOAuthParamByDefinitionIdOptional(null, requestBody.getSourceDefinitionId()) + .orElseGet(() -> new SourceOAuthParameter().withOauthParameterId(UUID.randomUUID())) + .withConfiguration(Jsons.jsonNode(requestBody.getParams())) + .withSourceDefinitionId(requestBody.getSourceDefinitionId()); + configRepository.writeSourceOAuthParam(param); + } + public void setDestinationInstancewideOauthParams(SetInstancewideDestinationOauthParamsRequestBody requestBody) throws JsonValidationException, IOException { - DestinationOAuthParameter param = new DestinationOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) + final DestinationOAuthParameter param = configRepository + .getDestinationOAuthParamByDefinitionIdOptional(null, requestBody.getDestinationDefinitionId()) + .orElseGet(() -> new DestinationOAuthParameter().withOauthParameterId(UUID.randomUUID())) .withConfiguration(Jsons.jsonNode(requestBody.getParams())) .withDestinationDefinitionId(requestBody.getDestinationDefinitionId()); configRepository.writeDestinationOAuthParam(param); } - public void setSourceInstancewideOauthParams(SetInstancewideSourceOauthParamsRequestBody requestBody) throws JsonValidationException, IOException { - SourceOAuthParameter param = new SourceOAuthParameter() - .withOauthParameterId(UUID.randomUUID()) - .withConfiguration(Jsons.jsonNode(requestBody.getParams())) - .withSourceDefinitionId(requestBody.getSourceDefinitionId()); - configRepository.writeSourceOAuthParam(param); - } - private OAuthFlowImplementation getSourceOAuthFlowImplementation(UUID sourceDefinitionId) throws JsonValidationException, ConfigNotFoundException, IOException { final StandardSourceDefinition standardSourceDefinition = configRepository diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 8299d8c59dde..d6a46858b754 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -26,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; +import io.airbyte.api.model.AuthSpecification; import io.airbyte.api.model.CheckConnectionRead; import io.airbyte.api.model.CheckConnectionRead.StatusEnum; import io.airbyte.api.model.ConnectionIdRequestBody; @@ -67,6 +68,7 @@ import io.airbyte.server.converters.CatalogConverter; import io.airbyte.server.converters.ConfigurationUpdate; import io.airbyte.server.converters.JobConverter; +import io.airbyte.server.converters.OauthModelConverter; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; @@ -246,11 +248,18 @@ public SourceDefinitionSpecificationRead getSourceDefinitionSpecification(Source final String imageName = DockerUtils.getTaggedImageName(source.getDockerRepository(), source.getDockerImageTag()); final SynchronousResponse response = getConnectorSpecification(imageName); final ConnectorSpecification spec = response.getOutput(); - return new SourceDefinitionSpecificationRead() + SourceDefinitionSpecificationRead specRead = new SourceDefinitionSpecificationRead() .jobInfo(JobConverter.getSynchronousJobRead(response)) .connectionSpecification(spec.getConnectionSpecification()) .documentationUrl(spec.getDocumentationUrl().toString()) .sourceDefinitionId(sourceDefinitionId); + + Optional authSpec = OauthModelConverter.getAuthSpec(spec); + if (authSpec.isPresent()) { + specRead.setAuthSpecification(authSpec.get()); + } + + return specRead; } public DestinationDefinitionSpecificationRead getDestinationSpecification(DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody) @@ -260,7 +269,8 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification(Destin final String imageName = DockerUtils.getTaggedImageName(destination.getDockerRepository(), destination.getDockerImageTag()); final SynchronousResponse response = getConnectorSpecification(imageName); final ConnectorSpecification spec = response.getOutput(); - return new DestinationDefinitionSpecificationRead() + + DestinationDefinitionSpecificationRead specRead = new DestinationDefinitionSpecificationRead() .jobInfo(JobConverter.getSynchronousJobRead(response)) .supportedDestinationSyncModes(Enums.convertListTo(spec.getSupportedDestinationSyncModes(), DestinationSyncMode.class)) .connectionSpecification(spec.getConnectionSpecification()) @@ -268,6 +278,13 @@ public DestinationDefinitionSpecificationRead getDestinationSpecification(Destin .supportsNormalization(spec.getSupportsNormalization()) .supportsDbt(spec.getSupportsDBT()) .destinationDefinitionId(destinationDefinitionId); + + Optional authSpec = OauthModelConverter.getAuthSpec(spec); + if (authSpec.isPresent()) { + specRead.setAuthSpecification(authSpec.get()); + } + + return specRead; } public SynchronousResponse getConnectorSpecification(String dockerImage) throws IOException { @@ -352,7 +369,7 @@ public JobInfoRead cancelJob(JobIdRequestBody jobIdRequestBody) throws IOExcepti private void cancelTemporalWorkflowIfPresent(long jobId) throws IOException { var latestAttemptId = jobPersistence.getJob(jobId).getAttempts().size() - 1; // attempts ids are monotonically increasing starting from 0 and - // specific to a job id, allowing us to do this. + // specific to a job id, allowing us to do this. var workflowId = jobPersistence.getAttemptTemporalWorkflowId(jobId, latestAttemptId); if (workflowId.isPresent()) { diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java index c13eedddd6fa..efae29d17fa8 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WebBackendConnectionsHandler.java @@ -26,6 +26,8 @@ import static java.util.stream.Collectors.toMap; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -39,6 +41,7 @@ import io.airbyte.api.model.ConnectionUpdate; import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationRead; +import io.airbyte.api.model.DestinationSearch; import io.airbyte.api.model.JobConfigType; import io.airbyte.api.model.JobListRequestBody; import io.airbyte.api.model.JobRead; @@ -51,10 +54,12 @@ import io.airbyte.api.model.SourceDiscoverSchemaRead; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceRead; +import io.airbyte.api.model.SourceSearch; import io.airbyte.api.model.WebBackendConnectionCreate; import io.airbyte.api.model.WebBackendConnectionRead; import io.airbyte.api.model.WebBackendConnectionReadList; import io.airbyte.api.model.WebBackendConnectionRequestBody; +import io.airbyte.api.model.WebBackendConnectionSearch; import io.airbyte.api.model.WebBackendConnectionUpdate; import io.airbyte.api.model.WebBackendOperationCreateOrUpdate; import io.airbyte.api.model.WorkspaceIdRequestBody; @@ -70,6 +75,7 @@ import java.util.Set; import java.util.UUID; import java.util.function.Predicate; +import org.apache.logging.log4j.util.Strings; public class WebBackendConnectionsHandler { @@ -172,6 +178,116 @@ private void setLatestSyncJobProperties(WebBackendConnectionRead WebBackendConne }); } + public WebBackendConnectionReadList webBackendSearchConnections(WebBackendConnectionSearch webBackendConnectionSearch) + throws ConfigNotFoundException, IOException, JsonValidationException { + + final List reads = Lists.newArrayList(); + for (ConnectionRead connectionRead : connectionsHandler.listConnections().getConnections()) { + if (matchSearch(webBackendConnectionSearch, connectionRead)) { + reads.add(buildWebBackendConnectionRead(connectionRead)); + } + } + + return new WebBackendConnectionReadList().connections(reads); + } + + private boolean matchSearch(WebBackendConnectionSearch connectionSearch, ConnectionRead connectionRead) + throws JsonValidationException, ConfigNotFoundException, IOException { + + final ConnectionRead connectionReadFromSearch = fromConnectionSearch(connectionSearch, connectionRead); + final SourceRead sourceRead = sourceHandler.getSource(new SourceIdRequestBody().sourceId(connectionRead.getSourceId())); + final SourceRead sourceReadFromSearch = fromSourceSearch(connectionSearch.getSource(), sourceRead); + final DestinationRead destinationRead = + destinationHandler.getDestination(new DestinationIdRequestBody().destinationId(connectionRead.getDestinationId())); + final DestinationRead destinationReadFromSearch = fromDestinationSearch(connectionSearch.getDestination(), destinationRead); + + return (connectionReadFromSearch == null || connectionReadFromSearch.equals(connectionRead)) && + (sourceReadFromSearch == null || sourceReadFromSearch.equals(sourceRead)) && + (destinationReadFromSearch == null || destinationReadFromSearch.equals(destinationRead)); + } + + private ConnectionRead fromConnectionSearch(WebBackendConnectionSearch connectionSearch, ConnectionRead connectionRead) { + if (connectionSearch == null) + return connectionRead; + + final ConnectionRead fromSearch = new ConnectionRead(); + fromSearch.connectionId(connectionSearch.getConnectionId() == null ? connectionRead.getConnectionId() : connectionSearch.getConnectionId()); + fromSearch.destinationId(connectionSearch.getDestinationId() == null ? connectionRead.getDestinationId() : connectionSearch.getDestinationId()); + fromSearch.name(Strings.isBlank(connectionSearch.getName()) ? connectionRead.getName() : connectionSearch.getName()); + fromSearch.namespaceFormat(Strings.isBlank(connectionSearch.getNamespaceFormat()) || connectionSearch.getNamespaceFormat().equals("null") + ? connectionRead.getNamespaceFormat() + : connectionSearch.getNamespaceFormat()); + fromSearch.namespaceDefinition( + connectionSearch.getNamespaceDefinition() == null ? connectionRead.getNamespaceDefinition() : connectionSearch.getNamespaceDefinition()); + fromSearch.prefix(Strings.isBlank(connectionSearch.getPrefix()) ? connectionRead.getPrefix() : connectionSearch.getPrefix()); + fromSearch.schedule(connectionSearch.getSchedule() == null ? connectionRead.getSchedule() : connectionSearch.getSchedule()); + fromSearch.sourceId(connectionSearch.getSourceId() == null ? connectionRead.getSourceId() : connectionSearch.getSourceId()); + fromSearch.status(connectionSearch.getStatus() == null ? connectionRead.getStatus() : connectionSearch.getStatus()); + + // these properties are not enabled in the search + fromSearch.resourceRequirements(connectionRead.getResourceRequirements()); + fromSearch.syncCatalog(connectionRead.getSyncCatalog()); + fromSearch.operationIds(connectionRead.getOperationIds()); + + return fromSearch; + } + + private SourceRead fromSourceSearch(SourceSearch sourceSearch, SourceRead sourceRead) { + if (sourceSearch == null) + return sourceRead; + + final SourceRead fromSearch = new SourceRead(); + fromSearch.name(Strings.isBlank(sourceSearch.getName()) ? sourceRead.getName() : sourceSearch.getName()); + fromSearch + .sourceDefinitionId(sourceSearch.getSourceDefinitionId() == null ? sourceRead.getSourceDefinitionId() : sourceSearch.getSourceDefinitionId()); + fromSearch.sourceId(sourceSearch.getSourceId() == null ? sourceRead.getSourceId() : sourceSearch.getSourceId()); + fromSearch.sourceName(Strings.isBlank(sourceSearch.getSourceName()) ? sourceRead.getSourceName() : sourceSearch.getSourceName()); + fromSearch.workspaceId(sourceSearch.getWorkspaceId() == null ? sourceRead.getWorkspaceId() : sourceSearch.getWorkspaceId()); + if (sourceSearch.getConnectionConfiguration() == null) { + fromSearch.connectionConfiguration(sourceRead.getConnectionConfiguration()); + } else { + JsonNode connectionConfiguration = sourceSearch.getConnectionConfiguration(); + sourceRead.getConnectionConfiguration().fieldNames() + .forEachRemaining(field -> { + if (!connectionConfiguration.has(field) && connectionConfiguration instanceof ObjectNode) { + ((ObjectNode) connectionConfiguration).set(field, sourceRead.getConnectionConfiguration().get(field)); + } + }); + fromSearch.connectionConfiguration(connectionConfiguration); + } + + return fromSearch; + } + + private DestinationRead fromDestinationSearch(DestinationSearch destinationSearch, DestinationRead destinationRead) { + if (destinationSearch == null) + return destinationRead; + + final DestinationRead fromSearch = new DestinationRead(); + fromSearch.name(Strings.isBlank(destinationSearch.getName()) ? destinationRead.getName() : destinationSearch.getName()); + fromSearch.destinationDefinitionId(destinationSearch.getDestinationDefinitionId() == null ? destinationRead.getDestinationDefinitionId() + : destinationSearch.getDestinationDefinitionId()); + fromSearch + .destinationId(destinationSearch.getDestinationId() == null ? destinationRead.getDestinationId() : destinationSearch.getDestinationId()); + fromSearch.destinationName( + Strings.isBlank(destinationSearch.getDestinationName()) ? destinationRead.getDestinationName() : destinationSearch.getDestinationName()); + fromSearch.workspaceId(destinationSearch.getWorkspaceId() == null ? destinationRead.getWorkspaceId() : destinationSearch.getWorkspaceId()); + if (destinationSearch.getConnectionConfiguration() == null) { + fromSearch.connectionConfiguration(destinationRead.getConnectionConfiguration()); + } else { + JsonNode connectionConfiguration = destinationSearch.getConnectionConfiguration(); + destinationRead.getConnectionConfiguration().fieldNames() + .forEachRemaining(field -> { + if (!connectionConfiguration.has(field) && connectionConfiguration instanceof ObjectNode) { + ((ObjectNode) connectionConfiguration).set(field, destinationRead.getConnectionConfiguration().get(field)); + } + }); + fromSearch.connectionConfiguration(connectionConfiguration); + } + + return fromSearch; + } + public WebBackendConnectionRead webBackendGetConnection(WebBackendConnectionRequestBody webBackendConnectionRequestBody) throws ConfigNotFoundException, IOException, JsonValidationException { final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody() diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java index 517da6215a34..f6953813c7cb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/WorkspacesHandler.java @@ -91,6 +91,7 @@ public WorkspaceRead createWorkspace(final WorkspaceCreate workspaceCreate) final Boolean anonymousDataCollection = workspaceCreate.getAnonymousDataCollection(); final Boolean news = workspaceCreate.getNews(); final Boolean securityUpdates = workspaceCreate.getSecurityUpdates(); + final Boolean displaySetupWizard = workspaceCreate.getDisplaySetupWizard(); final StandardWorkspace workspace = new StandardWorkspace() .withWorkspaceId(uuidSupplier.get()) @@ -101,7 +102,7 @@ public WorkspaceRead createWorkspace(final WorkspaceCreate workspaceCreate) .withAnonymousDataCollection(anonymousDataCollection != null ? anonymousDataCollection : false) .withNews(news != null ? news : false) .withSecurityUpdates(securityUpdates != null ? securityUpdates : false) - .withDisplaySetupWizard(false) + .withDisplaySetupWizard(displaySetupWizard != null ? displaySetupWizard : false) .withTombstone(false) .withNotifications(NotificationConverter.toConfigList(workspaceCreate.getNotifications())); diff --git a/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java b/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java index ccd968b3edeb..73c0bfae022a 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/ConfigDumpImporterTest.java @@ -190,7 +190,8 @@ public void testImportIntoWorkspaceWithConflicts() throws JsonValidationExceptio Jsons.clone(sourceConnection).withWorkspaceId(newWorkspaceId).withSourceId(not(eq(sourceConnection.getSourceId()))), eq(emptyConnectorSpec)); verify(configRepository).writeDestinationConnection( - Jsons.clone(destinationConnection).withWorkspaceId(newWorkspaceId).withDestinationId(not(eq(destinationConnection.getDestinationId())))); + Jsons.clone(destinationConnection).withWorkspaceId(newWorkspaceId).withDestinationId(not(eq(destinationConnection.getDestinationId()))), + eq(emptyConnectorSpec)); verify(configRepository) .writeStandardSyncOperation(Jsons.clone(operation).withWorkspaceId(newWorkspaceId).withOperationId(not(eq(operation.getOperationId())))); verify(configRepository).writeStandardSync(Jsons.clone(connection).withConnectionId(not(eq(connection.getConnectionId())))); @@ -241,7 +242,7 @@ public void testImportIntoWorkspaceWithoutConflicts() throws JsonValidationExcep verify(configRepository).writeSourceConnection( Jsons.clone(sourceConnection).withWorkspaceId(newWorkspaceId), emptyConnectorSpec); - verify(configRepository).writeDestinationConnection(Jsons.clone(destinationConnection).withWorkspaceId(newWorkspaceId)); + verify(configRepository).writeDestinationConnection(Jsons.clone(destinationConnection).withWorkspaceId(newWorkspaceId), emptyConnectorSpec); verify(configRepository).writeStandardSyncOperation(Jsons.clone(operation).withWorkspaceId(newWorkspaceId)); verify(configRepository).writeStandardSync(connection); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java index 988f28538f29..05b1381dda09 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionsHandlerTest.java @@ -347,6 +347,22 @@ void testListConnectionsForWorkspace() throws JsonValidationException, ConfigNot actualConnectionReadList.getConnections().get(0)); } + @Test + void testListConnections() throws JsonValidationException, ConfigNotFoundException, IOException { + when(configRepository.listStandardSyncs()) + .thenReturn(Lists.newArrayList(standardSync)); + when(configRepository.getSourceConnection(source.getSourceId())) + .thenReturn(source); + when(configRepository.getStandardSync(standardSync.getConnectionId())) + .thenReturn(standardSync); + + final ConnectionReadList actualConnectionReadList = connectionsHandler.listConnections(); + + assertEquals( + ConnectionHelpers.generateExpectedConnectionRead(standardSync), + actualConnectionReadList.getConnections().get(0)); + } + @Test void testDeleteConnection() throws JsonValidationException, IOException, ConfigNotFoundException { final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody().connectionId(standardSync.getConnectionId()); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java index 837b3c49f8cb..fb1c3673366b 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/DestinationHandlerTest.java @@ -71,7 +71,6 @@ class DestinationHandlerTest { private ConfigRepository configRepository; private StandardDestinationDefinition standardDestinationDefinition; private DestinationDefinitionSpecificationRead destinationDefinitionSpecificationRead; - private DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody; private DestinationConnection destinationConnection; private DestinationHandler destinationHandler; private ConnectionsHandler connectionsHandler; @@ -104,8 +103,8 @@ void setUp() throws IOException { imageName = DockerUtils.getTaggedImageName(standardDestinationDefinition.getDockerRepository(), standardDestinationDefinition.getDockerImageTag()); - destinationDefinitionIdRequestBody = - new DestinationDefinitionIdRequestBody().destinationDefinitionId(standardDestinationDefinition.getDestinationDefinitionId()); + DestinationDefinitionIdRequestBody destinationDefinitionIdRequestBody = new DestinationDefinitionIdRequestBody().destinationDefinitionId( + standardDestinationDefinition.getDestinationDefinitionId()); connectorSpecification = ConnectorSpecificationHelpers.generateConnectorSpecification(); @@ -154,7 +153,7 @@ void testCreateDestination() throws JsonValidationException, ConfigNotFoundExcep assertEquals(expectedDestinationRead, actualDestinationRead); verify(validator).ensure(destinationDefinitionSpecificationRead.getConnectionSpecification(), destinationConnection.getConfiguration()); - verify(configRepository).writeDestinationConnection(destinationConnection); + verify(configRepository).writeDestinationConnection(destinationConnection, connectorSpecification); verify(secretsProcessor) .maskSecrets(destinationConnection.getConfiguration(), destinationDefinitionSpecificationRead.getConnectionSpecification()); } @@ -181,7 +180,7 @@ void testDeleteDestination() throws JsonValidationException, ConfigNotFoundExcep destinationHandler.deleteDestination(destinationId); - verify(configRepository).writeDestinationConnection(expectedDestinationConnection); + verify(configRepository).writeDestinationConnection(expectedDestinationConnection, connectorSpecification); verify(connectionsHandler).listConnectionsForWorkspace(workspaceIdRequestBody); verify(connectionsHandler).deleteConnection(connectionRead); } @@ -225,7 +224,7 @@ void testUpdateDestination() throws JsonValidationException, ConfigNotFoundExcep assertEquals(expectedDestinationRead, actualDestinationRead); verify(secretsProcessor).maskSecrets(newConfiguration, destinationDefinitionSpecificationRead.getConnectionSpecification()); - verify(configRepository).writeDestinationConnection(expectedDestinationConnection); + verify(configRepository).writeDestinationConnection(expectedDestinationConnection, connectorSpecification); verify(validator).ensure(destinationDefinitionSpecificationRead.getConnectionSpecification(), newConfiguration); } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java index 34f883c3ff74..9cb62ce33685 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/OAuthHandlerTest.java @@ -25,6 +25,7 @@ package io.airbyte.server.handlers; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; import io.airbyte.api.model.SetInstancewideDestinationOauthParamsRequestBody; import io.airbyte.api.model.SetInstancewideSourceOauthParamsRequestBody; @@ -35,7 +36,9 @@ import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -53,6 +56,58 @@ public void init() { handler = new OAuthHandler(configRepository); } + @Test + void setSourceInstancewideOauthParams() throws JsonValidationException, IOException { + UUID sourceDefId = UUID.randomUUID(); + Map params = new HashMap<>(); + params.put("client_id", "123"); + params.put("client_secret", "hunter2"); + + SetInstancewideSourceOauthParamsRequestBody actualRequest = new SetInstancewideSourceOauthParamsRequestBody() + .sourceDefinitionId(sourceDefId) + .params(params); + + handler.setSourceInstancewideOauthParams(actualRequest); + + ArgumentCaptor argument = ArgumentCaptor.forClass(SourceOAuthParameter.class); + Mockito.verify(configRepository).writeSourceOAuthParam(argument.capture()); + assertEquals(Jsons.jsonNode(params), argument.getValue().getConfiguration()); + assertEquals(sourceDefId, argument.getValue().getSourceDefinitionId()); + } + + @Test + void resetSourceInstancewideOauthParams() throws JsonValidationException, IOException { + UUID sourceDefId = UUID.randomUUID(); + Map firstParams = new HashMap<>(); + firstParams.put("client_id", "123"); + firstParams.put("client_secret", "hunter2"); + SetInstancewideSourceOauthParamsRequestBody firstRequest = new SetInstancewideSourceOauthParamsRequestBody() + .sourceDefinitionId(sourceDefId) + .params(firstParams); + handler.setSourceInstancewideOauthParams(firstRequest); + + final UUID oauthParameterId = UUID.randomUUID(); + when(configRepository.getSourceOAuthParamByDefinitionIdOptional(null, sourceDefId)) + .thenReturn(Optional.of(new SourceOAuthParameter().withOauthParameterId(oauthParameterId))); + + Map secondParams = new HashMap<>(); + secondParams.put("client_id", "456"); + secondParams.put("client_secret", "hunter3"); + SetInstancewideSourceOauthParamsRequestBody secondRequest = new SetInstancewideSourceOauthParamsRequestBody() + .sourceDefinitionId(sourceDefId) + .params(secondParams); + handler.setSourceInstancewideOauthParams(secondRequest); + + ArgumentCaptor argument = ArgumentCaptor.forClass(SourceOAuthParameter.class); + Mockito.verify(configRepository, Mockito.times(2)).writeSourceOAuthParam(argument.capture()); + List capturedValues = argument.getAllValues(); + assertEquals(Jsons.jsonNode(firstParams), capturedValues.get(0).getConfiguration()); + assertEquals(Jsons.jsonNode(secondParams), capturedValues.get(1).getConfiguration()); + assertEquals(sourceDefId, capturedValues.get(0).getSourceDefinitionId()); + assertEquals(sourceDefId, capturedValues.get(1).getSourceDefinitionId()); + assertEquals(oauthParameterId, capturedValues.get(1).getOauthParameterId()); + } + @Test void setDestinationInstancewideOauthParams() throws JsonValidationException, IOException { UUID destinationDefId = UUID.randomUUID(); @@ -73,22 +128,36 @@ void setDestinationInstancewideOauthParams() throws JsonValidationException, IOE } @Test - void setSourceInstancewideOauthParams() throws JsonValidationException, IOException { - UUID sourceDefId = UUID.randomUUID(); - Map params = new HashMap<>(); - params.put("client_id", "123"); - params.put("client_secret", "hunter2"); + void resetDestinationInstancewideOauthParams() throws JsonValidationException, IOException { + UUID destinationDefId = UUID.randomUUID(); + Map firstParams = new HashMap<>(); + firstParams.put("client_id", "123"); + firstParams.put("client_secret", "hunter2"); + SetInstancewideDestinationOauthParamsRequestBody firstRequest = new SetInstancewideDestinationOauthParamsRequestBody() + .destinationDefinitionId(destinationDefId) + .params(firstParams); + handler.setDestinationInstancewideOauthParams(firstRequest); - SetInstancewideSourceOauthParamsRequestBody actualRequest = new SetInstancewideSourceOauthParamsRequestBody() - .sourceDefinitionId(sourceDefId) - .params(params); + final UUID oauthParameterId = UUID.randomUUID(); + when(configRepository.getDestinationOAuthParamByDefinitionIdOptional(null, destinationDefId)) + .thenReturn(Optional.of(new DestinationOAuthParameter().withOauthParameterId(oauthParameterId))); - handler.setSourceInstancewideOauthParams(actualRequest); + Map secondParams = new HashMap<>(); + secondParams.put("client_id", "456"); + secondParams.put("client_secret", "hunter3"); + SetInstancewideDestinationOauthParamsRequestBody secondRequest = new SetInstancewideDestinationOauthParamsRequestBody() + .destinationDefinitionId(destinationDefId) + .params(secondParams); + handler.setDestinationInstancewideOauthParams(secondRequest); - ArgumentCaptor argument = ArgumentCaptor.forClass(SourceOAuthParameter.class); - Mockito.verify(configRepository).writeSourceOAuthParam(argument.capture()); - assertEquals(Jsons.jsonNode(params), argument.getValue().getConfiguration()); - assertEquals(sourceDefId, argument.getValue().getSourceDefinitionId()); + ArgumentCaptor argument = ArgumentCaptor.forClass(DestinationOAuthParameter.class); + Mockito.verify(configRepository, Mockito.times(2)).writeDestinationOAuthParam(argument.capture()); + List capturedValues = argument.getAllValues(); + assertEquals(Jsons.jsonNode(firstParams), capturedValues.get(0).getConfiguration()); + assertEquals(Jsons.jsonNode(secondParams), capturedValues.get(1).getConfiguration()); + assertEquals(destinationDefId, capturedValues.get(0).getDestinationDefinitionId()); + assertEquals(destinationDefId, capturedValues.get(1).getDestinationDefinitionId()); + assertEquals(oauthParameterId, capturedValues.get(1).getOauthParameterId()); } } diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java index d0d298514e65..6dfda282b1b7 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/WebBackendConnectionsHandlerTest.java @@ -46,6 +46,7 @@ import io.airbyte.api.model.ConnectionUpdate; import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationRead; +import io.airbyte.api.model.DestinationSearch; import io.airbyte.api.model.DestinationSyncMode; import io.airbyte.api.model.JobConfigType; import io.airbyte.api.model.JobInfoRead; @@ -62,16 +63,19 @@ import io.airbyte.api.model.SourceDiscoverSchemaRead; import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceRead; +import io.airbyte.api.model.SourceSearch; import io.airbyte.api.model.SyncMode; import io.airbyte.api.model.SynchronousJobRead; import io.airbyte.api.model.WebBackendConnectionCreate; import io.airbyte.api.model.WebBackendConnectionRead; import io.airbyte.api.model.WebBackendConnectionReadList; import io.airbyte.api.model.WebBackendConnectionRequestBody; +import io.airbyte.api.model.WebBackendConnectionSearch; import io.airbyte.api.model.WebBackendConnectionUpdate; import io.airbyte.api.model.WebBackendOperationCreateOrUpdate; import io.airbyte.api.model.WorkspaceIdRequestBody; import io.airbyte.commons.enums.Enums; +import io.airbyte.commons.json.Jsons; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardDestinationDefinition; @@ -247,6 +251,71 @@ public void testWebBackendListConnectionsForWorkspace() throws ConfigNotFoundExc assertEquals(expected, WebBackendConnectionReadList.getConnections().get(0)); } + @Test + public void testWebBackendSearchConnections() throws ConfigNotFoundException, IOException, JsonValidationException { + final WorkspaceIdRequestBody workspaceIdRequestBody = new WorkspaceIdRequestBody(); + workspaceIdRequestBody.setWorkspaceId(sourceRead.getWorkspaceId()); + + final ConnectionReadList connectionReadList = new ConnectionReadList(); + connectionReadList.setConnections(Collections.singletonList(connectionRead)); + final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody(); + connectionIdRequestBody.setConnectionId(connectionRead.getConnectionId()); + when(connectionsHandler.listConnections()).thenReturn(connectionReadList); + when(operationsHandler.listOperationsForConnection(connectionIdRequestBody)).thenReturn(operationReadList); + + final WebBackendConnectionSearch webBackendConnectionSearch = new WebBackendConnectionSearch(); + WebBackendConnectionReadList WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + assertEquals(expected, WebBackendConnectionReadList.getConnections().get(0)); + + webBackendConnectionSearch.setSource(new SourceSearch().sourceId(UUID.randomUUID())); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(0, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.setSource(new SourceSearch().sourceId(connectionRead.getSourceId())); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + + final DestinationSearch destinationSearch = new DestinationSearch(); + webBackendConnectionSearch.setDestination(destinationSearch); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + + destinationSearch.connectionConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "not-found"))); + webBackendConnectionSearch.setDestination(destinationSearch); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(0, WebBackendConnectionReadList.getConnections().size()); + + destinationSearch.connectionConfiguration(Jsons.jsonNode(Collections.singletonMap("apiKey", "123-abc"))); + webBackendConnectionSearch.setDestination(destinationSearch); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.name("not-existent"); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(0, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.name(connectionRead.getName()); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.namespaceDefinition(NamespaceDefinitionType.CUSTOMFORMAT); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(0, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.namespaceDefinition(connectionRead.getNamespaceDefinition()); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.status(ConnectionStatus.INACTIVE); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(0, WebBackendConnectionReadList.getConnections().size()); + + webBackendConnectionSearch.status(ConnectionStatus.ACTIVE); + WebBackendConnectionReadList = wbHandler.webBackendSearchConnections(webBackendConnectionSearch); + assertEquals(1, WebBackendConnectionReadList.getConnections().size()); + } + @Test public void testWebBackendGetConnection() throws ConfigNotFoundException, IOException, JsonValidationException { final ConnectionIdRequestBody connectionIdRequestBody = new ConnectionIdRequestBody(); diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java index e6b4bf7b3909..ea081106f2ef 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/MigrationAcceptanceTest.java @@ -98,7 +98,6 @@ private Consumer logConsumerForServer(Set expectedLogs) { return logLine -> expectedLogs.removeIf(entry -> { if (logLine.contains("Migrating from version")) { System.out.println("logLine = " + logLine); - System.out.println("logLine = " + logLine); } return logLine.contains(entry); }); diff --git a/airbyte-webapp/.env b/airbyte-webapp/.env index 225e62d8727b..24e53e9e4c04 100644 --- a/airbyte-webapp/.env +++ b/airbyte-webapp/.env @@ -2,4 +2,4 @@ REACT_APP_SEGMENT_TOKEN=6cxNSmQyGSKcATLdJ2pL6WsawkzEMDAN REACT_APP_FULL_STORY_ORG=13AXQ4 REACT_APP_PAPERCUPS_ACCOUNT_ID=74560291-451e-4ceb-a802-56706ece528b REACT_APP_OPEN_REPLAY_PROJECT_ID=6611843272536134 -REACT_APP_SENTRY_DNS= \ No newline at end of file +REACT_APP_SENTRY_DNS= diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index afbb815221bc..652ee378d86f 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.29.17-alpha", + "version": "0.29.21-alpha", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 42eda350ee3e..674a0fed49bc 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.29.17-alpha", + "version": "0.29.21-alpha", "private": true, "scripts": { "start": "react-scripts start", diff --git a/airbyte-webapp/public/newsletter.png b/airbyte-webapp/public/newsletter.png index f265def92e11..f49f174b00f4 100644 Binary files a/airbyte-webapp/public/newsletter.png and b/airbyte-webapp/public/newsletter.png differ diff --git a/airbyte-webapp/public/play.svg b/airbyte-webapp/public/play.svg new file mode 100644 index 000000000000..a6976352ad28 --- /dev/null +++ b/airbyte-webapp/public/play.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/airbyte-webapp/public/process-arrow.svg b/airbyte-webapp/public/process-arrow.svg new file mode 100644 index 000000000000..1258bc739c8a --- /dev/null +++ b/airbyte-webapp/public/process-arrow.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/public/rectangle.svg b/airbyte-webapp/public/rectangle.svg new file mode 100644 index 000000000000..66aa72f35d11 --- /dev/null +++ b/airbyte-webapp/public/rectangle.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/public/rocket.png b/airbyte-webapp/public/rocket.png new file mode 100644 index 000000000000..c5cf200f9d46 Binary files /dev/null and b/airbyte-webapp/public/rocket.png differ diff --git a/airbyte-webapp/public/stars-background.svg b/airbyte-webapp/public/stars-background.svg new file mode 100644 index 000000000000..e90bb65949d0 --- /dev/null +++ b/airbyte-webapp/public/stars-background.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/airbyte-webapp/public/video-background.svg b/airbyte-webapp/public/video-background.svg new file mode 100644 index 000000000000..be3171af5585 --- /dev/null +++ b/airbyte-webapp/public/video-background.svg @@ -0,0 +1,3 @@ + + + diff --git a/airbyte-webapp/public/videoCover.png b/airbyte-webapp/public/videoCover.png new file mode 100644 index 000000000000..402cdfb7e16d Binary files /dev/null and b/airbyte-webapp/public/videoCover.png differ diff --git a/airbyte-webapp/src/App.tsx b/airbyte-webapp/src/App.tsx index 46503702534a..74f1740edd08 100644 --- a/airbyte-webapp/src/App.tsx +++ b/airbyte-webapp/src/App.tsx @@ -17,6 +17,7 @@ import { usePickFirstWorkspace, } from "hooks/services/useWorkspace"; import { Feature, FeatureService } from "hooks/services/Feature"; +import { OnboardingServiceProvider } from "hooks/services/Onboarding"; import { ServicesProvider } from "core/servicesProvider"; import { useApiServices } from "core/defaultServices"; import { envConfigProvider, windowConfigProvider } from "./config"; @@ -97,7 +98,9 @@ const App: React.FC = () => { - + + + diff --git a/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx b/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx index 570c953f17fb..1087e426b119 100644 --- a/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx +++ b/airbyte-webapp/src/components/CenteredPageComponents/BigButton.tsx @@ -1,11 +1,13 @@ import styled from "styled-components"; import { Button } from "components"; -const BigButton = styled(Button)` +const BigButton = styled(Button)<{ shadow?: boolean }>` font-size: 16px; line-height: 19px; padding: 10px 27px; font-weight: 500; + box-shadow: ${({ shadow }) => + shadow ? "0 8px 5px -5px rgba(0, 0, 0, 0.2)" : "none"}; `; export default BigButton; diff --git a/airbyte-webapp/src/components/ContentCard/ContentCard.tsx b/airbyte-webapp/src/components/ContentCard/ContentCard.tsx index 873fe5a7efef..e731d0a54474 100644 --- a/airbyte-webapp/src/components/ContentCard/ContentCard.tsx +++ b/airbyte-webapp/src/components/ContentCard/ContentCard.tsx @@ -7,6 +7,7 @@ type IProps = { title?: string | React.ReactNode; className?: string; onClick?: () => void; + full?: boolean; }; const Title = styled(H5)` @@ -19,7 +20,7 @@ const Title = styled(H5)` `; const ContentCard: React.FC = (props) => ( - + {props.title ? {props.title} : null} {props.children} diff --git a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx b/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx index ae181e28dbe7..531f96cd230d 100644 --- a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx +++ b/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx @@ -40,6 +40,7 @@ type IProps = { source: Source; destination: Destination; afterSubmitConnection?: () => void; + noTitles?: boolean; }; const CreateConnectionContent: React.FC = ({ @@ -47,6 +48,7 @@ const CreateConnectionContent: React.FC = ({ destination, afterSubmitConnection, additionBottomControls, + noTitles, }) => { const { createConnection } = useConnection(); const analyticsService = useAnalytics(); @@ -80,7 +82,11 @@ const CreateConnectionContent: React.FC = ({ if (isLoading) { return ( - }> + + } + > ); @@ -88,7 +94,11 @@ const CreateConnectionContent: React.FC = ({ if (schemaErrorStatus) { return ( - }> + + } + > {additionBottomControls}} @@ -130,7 +140,11 @@ const CreateConnectionContent: React.FC = ({ }; return ( - }> + + } + > }> void; + clear?: boolean; + closeOnBackground?: boolean; }; const fadeIn = keyframes` @@ -27,7 +29,13 @@ const Overlay = styled.div` z-index: 10; `; -const Modal: React.FC = ({ children, title, onClose }) => { +const Modal: React.FC = ({ + children, + title, + onClose, + clear, + closeOnBackground, +}) => { const handleUserKeyPress = useCallback((event, closeModal) => { const { keyCode } = event; if (keyCode === 27) { @@ -50,8 +58,8 @@ const Modal: React.FC = ({ children, title, onClose }) => { }, [handleUserKeyPress, onClose]); return createPortal( - - {children} + (closeOnBackground && onClose ? onClose() : null)}> + {clear ? children : {children}} , document.body ); diff --git a/airbyte-webapp/src/components/base/Card/Card.tsx b/airbyte-webapp/src/components/base/Card/Card.tsx index 22aae4ba0826..783563ad89bb 100644 --- a/airbyte-webapp/src/components/base/Card/Card.tsx +++ b/airbyte-webapp/src/components/base/Card/Card.tsx @@ -1,6 +1,7 @@ import styled from "styled-components"; -export const Card = styled.div` +export const Card = styled.div<{ full?: boolean }>` + width: ${({ full }) => (full ? "100%" : "auto")}; background: ${({ theme }) => theme.whiteColor}; border-radius: 10px; box-shadow: 0 2px 4px ${({ theme }) => theme.cardShadowColor}; diff --git a/airbyte-webapp/src/components/base/Titles/Titles.tsx b/airbyte-webapp/src/components/base/Titles/Titles.tsx index c3781261e6fc..4ba492bf272b 100644 --- a/airbyte-webapp/src/components/base/Titles/Titles.tsx +++ b/airbyte-webapp/src/components/base/Titles/Titles.tsx @@ -3,6 +3,7 @@ import styled from "styled-components"; type IProps = { center?: boolean; bold?: boolean; + parentColor?: boolean; }; export const H1 = styled.h1` @@ -12,7 +13,8 @@ export const H1 = styled.h1` font-weight: ${(props) => (props.bold ? 600 : 500)}; display: block; text-align: ${(props) => (props.center ? "center" : "left")}; - color: ${({ theme }) => theme.textColor}; + color: ${({ theme, parentColor }) => + parentColor ? "inherid" : theme.textColor}; margin: 0; `; diff --git a/airbyte-webapp/src/config/casesConfig.json b/airbyte-webapp/src/config/casesConfig.json new file mode 100644 index 000000000000..66c30786c5c8 --- /dev/null +++ b/airbyte-webapp/src/config/casesConfig.json @@ -0,0 +1,7 @@ +[ + "replicateMySQL", + "consolidateMarketing", + "consolidatePayment", + "buildDashboard", + "zoomCalls" +] diff --git a/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts b/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts index f63cb20648a9..cb352d8b266a 100644 --- a/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts +++ b/airbyte-webapp/src/core/domain/catalog/fieldUtil.ts @@ -34,8 +34,7 @@ const traverseSchemaToField = ( const traverseJsonSchemaProperties = ( jsonSchema: JSONSchema7Definition, key: string, - path: string = key, - depth = 0 + path: string[] = [] ): SyncSchemaField[] => { if (typeof jsonSchema === "boolean") { return []; @@ -45,12 +44,7 @@ const traverseJsonSchemaProperties = ( if (jsonSchema.properties) { fields = Object.entries(jsonSchema.properties) .flatMap(([k, schema]) => - traverseJsonSchemaProperties( - schema, - k, - depth === 0 ? k : `${path}.${k}`, - depth + 1 - ) + traverseJsonSchemaProperties(schema, k, [...path, k]) ) .flat(2); } @@ -58,7 +52,7 @@ const traverseJsonSchemaProperties = ( return [ { cleanedName: key, - name: path, + path, key, fields, type: diff --git a/airbyte-webapp/src/core/domain/catalog/models.ts b/airbyte-webapp/src/core/domain/catalog/models.ts index cb99d68797dd..965b4f3bccc2 100644 --- a/airbyte-webapp/src/core/domain/catalog/models.ts +++ b/airbyte-webapp/src/core/domain/catalog/models.ts @@ -1,8 +1,8 @@ export type SyncSchemaField = { - name: string; cleanedName: string; type: string; key: string; + path: string[]; fields?: SyncSchemaField[]; }; diff --git a/airbyte-webapp/src/hooks/services/Onboarding/OnboardingService.tsx b/airbyte-webapp/src/hooks/services/Onboarding/OnboardingService.tsx new file mode 100644 index 000000000000..951eef8fcf02 --- /dev/null +++ b/airbyte-webapp/src/hooks/services/Onboarding/OnboardingService.tsx @@ -0,0 +1,55 @@ +import React, { useContext, useMemo } from "react"; +import { useLocalStorage } from "react-use"; +import useWorkspace from "hooks/services/useWorkspace"; +import casesConfig from "config/casesConfig.json"; + +type Context = { + feedbackPassed?: boolean; + passFeedback: () => void; + useCases?: string[]; + skipCase: (skipId: string) => void; +}; + +export const OnboardingServiceContext = React.createContext( + null +); + +export const OnboardingServiceProvider: React.FC = ({ children }) => { + const { workspace } = useWorkspace(); + const [feedbackPassed, setFeedbackPassed] = useLocalStorage( + `${workspace.workspaceId}/passFeedback`, + false + ); + const [useCases, setUseCases] = useLocalStorage( + `${workspace.workspaceId}/useCases`, + casesConfig + ); + + const ctx = useMemo( + () => ({ + feedbackPassed, + passFeedback: () => setFeedbackPassed(true), + useCases, + skipCase: (skipId: string) => + setUseCases(useCases?.filter((item) => item !== skipId)), + }), + [feedbackPassed, useCases] + ); + + return ( + + {children} + + ); +}; + +export const useOnboardingService = (): Context => { + const onboardingService = useContext(OnboardingServiceContext); + if (!onboardingService) { + throw new Error( + "useOnboardingService must be used within a OnboardingServiceProvider." + ); + } + + return onboardingService; +}; diff --git a/airbyte-webapp/src/hooks/services/Onboarding/index.tsx b/airbyte-webapp/src/hooks/services/Onboarding/index.tsx new file mode 100644 index 000000000000..305b4ce97d08 --- /dev/null +++ b/airbyte-webapp/src/hooks/services/Onboarding/index.tsx @@ -0,0 +1 @@ +export * from "./OnboardingService"; diff --git a/airbyte-webapp/src/hooks/services/useConnectionHook.tsx b/airbyte-webapp/src/hooks/services/useConnectionHook.tsx index d3c929a69448..78fd1ea8a396 100644 --- a/airbyte-webapp/src/hooks/services/useConnectionHook.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectionHook.tsx @@ -17,12 +17,15 @@ import { SyncSchema } from "core/domain/catalog"; import { SourceDefinition } from "core/resources/SourceDefinition"; import { Source } from "core/resources/Source"; import { Routes } from "pages/routes"; -import useRouter from "../useRouter"; import { Destination } from "core/resources/Destination"; import useWorkspace from "./useWorkspace"; import { Operation } from "core/domain/connection/operation"; -import { equal } from "utils/objects"; import { useAnalytics } from "hooks/useAnalytics"; +import useRouter from "hooks/useRouter"; +import { useGetService } from "core/servicesProvider"; +import { RequestMiddleware } from "core/request/RequestMiddleware"; + +import { equal } from "utils/objects"; export type ValuesProps = { schedule: ScheduleProperties | null; @@ -65,8 +68,13 @@ type UpdateStateConnection = { function useConnectionService(): ConnectionService { const config = useConfig(); + const middlewares = useGetService( + "DefaultRequestMiddlewares" + ); - return useMemo(() => new ConnectionService(config.apiUrl), [config]); + return useMemo(() => new ConnectionService(config.apiUrl, middlewares), [ + config, + ]); } export const useConnectionLoad = ( @@ -95,10 +103,11 @@ const useConnection = (): { updateConnection: (conn: UpdateConnection) => Promise; updateStateConnection: (conn: UpdateStateConnection) => Promise; resetConnection: (connId: string) => Promise; + syncConnection: (conn: Connection) => Promise; deleteConnection: (payload: { connectionId: string }) => Promise; } => { const { push } = useRouter(); - const { finishOnboarding, workspace } = useWorkspace(); + const { workspace } = useWorkspace(); const analyticsService = useAnalytics(); const createConnectionResource = useFetcher(ConnectionResource.createShape()); @@ -108,6 +117,7 @@ const useConnection = (): { ); const deleteConnectionResource = useFetcher(ConnectionResource.deleteShape()); const resetConnectionResource = useFetcher(ConnectionResource.reset()); + const syncConnectionResource = useFetcher(ConnectionResource.syncShape()); const createConnection = async ({ values, @@ -155,9 +165,6 @@ const useConnection = (): { connector_destination_definition_id: destinationDefinition?.destinationDefinitionId, }); - if (workspace.displaySetupWizard) { - await finishOnboarding(); - } return result; } catch (e) { @@ -221,12 +228,37 @@ const useConnection = (): { [resetConnectionResource] ); + const syncConnection = async (connection: Connection) => { + analyticsService.track("Source - Action", { + action: "Full refresh sync", + connector_source: connection.source?.sourceName, + connector_source_id: connection.source?.sourceDefinitionId, + connector_destination: connection.destination?.name, + connector_destination_definition_id: + connection.destination?.destinationDefinitionId, + frequency: connection.schedule, + }); + await syncConnectionResource({ + connectionId: connection.connectionId, + }); + }; + return { createConnection, updateConnection, updateStateConnection, resetConnection, deleteConnection, + syncConnection, }; }; + +const useConnectionList = (): { connections: Connection[] } => { + const { workspace } = useWorkspace(); + return useResource(ConnectionResource.listShape(), { + workspaceId: workspace.workspaceId, + }); +}; + +export { useConnectionList }; export default useConnection; diff --git a/airbyte-webapp/src/hooks/services/useDestinationHook.tsx b/airbyte-webapp/src/hooks/services/useDestinationHook.tsx index 7222b62bf030..57107055fb6d 100644 --- a/airbyte-webapp/src/hooks/services/useDestinationHook.tsx +++ b/airbyte-webapp/src/hooks/services/useDestinationHook.tsx @@ -27,7 +27,7 @@ export const useDestinationDefinitionSpecificationLoad = ( ): { isLoading: boolean; destinationDefinitionSpecification?: DestinationDefinitionSpecification; - error?: Error; + sourceDefinitionError?: Error; } => { const { loading: isLoading, @@ -42,7 +42,11 @@ export const useDestinationDefinitionSpecificationLoad = ( : null ); - return { destinationDefinitionSpecification, error, isLoading }; + return { + destinationDefinitionSpecification, + sourceDefinitionError: error, + isLoading, + }; }; export const useDestinationDefinitionSpecificationLoadAsync = ( diff --git a/airbyte-webapp/src/hooks/services/useSourceHook.tsx b/airbyte-webapp/src/hooks/services/useSourceHook.tsx index 17ec8d7d41eb..fbe471f2bba1 100644 --- a/airbyte-webapp/src/hooks/services/useSourceHook.tsx +++ b/airbyte-webapp/src/hooks/services/useSourceHook.tsx @@ -28,12 +28,12 @@ export const useSourceDefinitionSpecificationLoad = ( sourceDefinitionId: string ): { isLoading: boolean; - error?: Error; + sourceDefinitionError?: Error; sourceDefinitionSpecification?: SourceDefinitionSpecification; } => { const { loading: isLoading, - error, + error: sourceDefinitionError, data: sourceDefinitionSpecification, } = useStatefulResource( SourceDefinitionSpecificationResource.detailShape(), @@ -44,7 +44,7 @@ export const useSourceDefinitionSpecificationLoad = ( : null ); - return { sourceDefinitionSpecification, error, isLoading }; + return { sourceDefinitionSpecification, sourceDefinitionError, isLoading }; }; type SourceService = { diff --git a/airbyte-webapp/src/hooks/services/useWorkspace.tsx b/airbyte-webapp/src/hooks/services/useWorkspace.tsx index f7700c123e4a..1f1ca3ea33e9 100644 --- a/airbyte-webapp/src/hooks/services/useWorkspace.tsx +++ b/airbyte-webapp/src/hooks/services/useWorkspace.tsx @@ -6,6 +6,8 @@ import NotificationsResource, { } from "core/resources/Notifications"; import { useGetService } from "core/servicesProvider"; import { useAnalytics } from "../useAnalytics"; +import { Source } from "core/resources/Source"; +import { Destination } from "core/resources/Destination"; export const usePickFirstWorkspace = (): Workspace => { const { workspaces } = useResource(WorkspaceResource.listShape(), {}); @@ -44,6 +46,15 @@ const useWorkspace = (): { securityUpdates: boolean; }) => Promise; finishOnboarding: (skipStep?: string) => Promise; + sendFeedback: ({ + feedback, + source, + destination, + }: { + feedback: string; + source: Source; + destination: Destination; + }) => Promise; } => { const updateWorkspace = useFetcher(WorkspaceResource.updateShape()); const tryWebhookUrl = useFetcher(NotificationsResource.tryShape()); @@ -71,6 +82,24 @@ const useWorkspace = (): { ); }; + const sendFeedback = async ({ + feedback, + source, + destination, + }: { + feedback: string; + source: Source; + destination: Destination; + }) => { + analyticsService.track("Onboarding Feedback", { + feedback, + connector_source_definition: source?.sourceName, + connector_source_definition_id: source?.sourceDefinitionId, + connector_destination_definition: destination?.destinationName, + connector_destination_definition_id: destination?.destinationDefinitionId, + }); + }; + const setInitialSetupConfig = async (data: { email: string; anonymousDataCollection: boolean; @@ -147,6 +176,7 @@ const useWorkspace = (): { updatePreferences, updateWebhook, testWebhook, + sendFeedback, }; }; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 69be4a569625..9f6861d3d492 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -11,6 +11,7 @@ "sidebar.connections": "Connections", "sidebar.settings": "Settings", "sidebar.update": "Update", + "sidebar.onboarding": "Onboarding", "form.continue": "Continue", "form.yourEmail": "Your email", @@ -54,6 +55,7 @@ "form.testingConnection": "Testing connection...", "form.successTests": "All connection tests passed!", "form.failedTests": "The connection tests failed.", + "form.failedFetchingConnector": "Fetching connector failed.", "form.filter_dbs": "Filter DBs", "form.serviceAccount": "Service Account", "form.validate_records": "Validate Records", @@ -167,6 +169,34 @@ "onboarding.fetchingSchema": "We are fetching the schema of your data source. \nThis should take less than a minute, but may take a few minutes on slow internet connections or data sources with a large amount of tables.", "onboarding.tutorial": "Check how you can sync PostgreSQL databases in minutes", "onboarding.skipOnboarding": "Skip Onboarding", + "onboarding.welcome": "Welcome to Airbyte!", + "onboarding.welcomeUser": "Welcome to Airbyte, {name}!", + "onboarding.welcomeUser.text": "Your path to syncing your data starts here.Connections are automated data pipelines that replicate data from a source to a destination. ", + "onboarding.or": "or", + "onboarding.watchVideo": "Watch the 2-min demo video", + "onboarding.exploreDemo": "Explore our demo app with test data", + "onboarding.firstConnection": "Set up your first connection", + "onboarding.createFirstSource": "Create your first source", + "onboarding.createFirstSource.text": "Sources are tools where the data will be replicated from. ", + "onboarding.createFirstDestination": "Create your first destination", + "onboarding.createFirstDestination.text": "Sources are tools where the data will be replicated from. ", + "onboarding.createConnection": "Set up the connection", + "onboarding.createConnection.text": "Sources are tools where the data will be replicated from. ", + "onboarding.synchronisationProgress": "SourceDestination = Synchronisation in progress", + "onboarding.useCases": "Enable popular use cases", + "onboarding.replicateMySQL": "Replicate your MySQL database to Postgres with log-based CDC", + "onboarding.consolidateMarketing": "Consolidate your marketing data to compute the CAC for your paid customers", + "onboarding.consolidatePayment": "Consolidate your payment data to compute your LTV", + "onboarding.buildDashboard": "Build an activity dashboard for your engineering project", + "onboarding.zoomCalls": "Visualize the time spent by your team in Zoom calls ", + "onboarding.skip": "Skip", + "onboarding.closeOnboarding": "Close onboarding", + "onboarding.syncCompleted": "Your first sync has been successfully completed!", + "onboarding.checkData": "Please check the data at the destination.\nDoes it fit with your expectations?", + "onboarding.skipNow": "Skip for now", + "onboarding.firstSync": "Start your first sync", + "onboarding.syncFailed": "Your sync is failed. Please try again", + "onboarding.startAgain": "Your sync was cancelled. You can start it again", "sources.searchIncremental": "Search cursor value for incremental", "sources.incrementalDefault": "{value} (default)", diff --git a/airbyte-webapp/src/packages/cloud/routes.tsx b/airbyte-webapp/src/packages/cloud/routes.tsx index 6d3b08cd0c4f..ba63df6b10f1 100644 --- a/airbyte-webapp/src/packages/cloud/routes.tsx +++ b/airbyte-webapp/src/packages/cloud/routes.tsx @@ -36,9 +36,11 @@ import { PageConfig } from "pages/SettingsPage/SettingsPage"; import { WorkspaceSettingsView } from "./views/workspaces/WorkspaceSettingsView"; import { UsersSettingsView } from "packages/cloud/views/users/UsersSettingsView/UsersSettingsView"; import { AccountSettingsView } from "packages/cloud/views/users/AccountSettingsView/AccountSettingsView"; +import OnboardingPage from "pages/OnboardingPage"; import { ConfirmEmailPage } from "./views/auth/ConfirmEmailPage"; import useRouter from "hooks/useRouter"; import { WithPageAnalytics } from "pages/withPageAnalytics"; +import useWorkspace from "../../hooks/services/useWorkspace"; export enum Routes { Preferences = "/preferences", @@ -75,6 +77,7 @@ const MainRoutes: React.FC<{ currentWorkspaceId: string }> = ({ }) => { useGetWorkspace(currentWorkspaceId); const { countNewSourceVersion, countNewDestinationVersion } = useConnector(); + const { workspace } = useWorkspace(); const pageConfig = useMemo( () => ({ @@ -145,6 +148,11 @@ const MainRoutes: React.FC<{ currentWorkspaceId: string }> = ({ + {workspace.displaySetupWizard && ( + + + + )} diff --git a/airbyte-webapp/src/packages/cloud/services/useDefaultRequestMiddlewares.tsx b/airbyte-webapp/src/packages/cloud/services/useDefaultRequestMiddlewares.tsx index 9ab2a4d103d8..20cc9b381831 100644 --- a/airbyte-webapp/src/packages/cloud/services/useDefaultRequestMiddlewares.tsx +++ b/airbyte-webapp/src/packages/cloud/services/useDefaultRequestMiddlewares.tsx @@ -5,5 +5,5 @@ import { useGetService } from "core/servicesProvider"; * This hook is responsible for registering RequestMiddlewares used in BaseRequest */ export const useDefaultRequestMiddlewares = (): RequestMiddleware[] => { - return useGetService("DefaultRequestMiddlewares"); + return useGetService("DefaultRequestMiddlewares"); }; diff --git a/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts b/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts new file mode 100644 index 000000000000..8643259af230 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts @@ -0,0 +1,12 @@ +import { useMutation } from "react-query"; + +import { useGetUserService } from "./UserService"; + +export const useUserHook = (onSuccess: () => void, onError: () => void) => { + const service = useGetUserService(); + + return useMutation(async (id: string) => service.remove(id), { + onSuccess, + onError, + }); +}; diff --git a/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx b/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx index 5937522dd06e..e2f6f02dee83 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/SideBar/SideBar.tsx @@ -16,7 +16,9 @@ import Indicator from "components/Indicator"; import Source from "views/layout/SideBar/components/SourceIcon"; import Connections from "views/layout/SideBar/components/ConnectionsIcon"; import Destination from "views/layout/SideBar/components/DestinationIcon"; +import Onboarding from "views/layout/SideBar/components/OnboardingIcon"; import { WorkspacePopout } from "packages/cloud/views/workspaces/WorkspacePopout"; +import useWorkspace from "hooks/services/useWorkspace"; const Bar = styled.nav` width: 100px; @@ -123,6 +125,7 @@ const WorkspaceButton = styled.div` const SideBar: React.FC = () => { const { hasNewVersions } = useConnector(); const config = useConfig(); + const { workspace } = useWorkspace(); return ( @@ -136,6 +139,16 @@ const SideBar: React.FC = () => { )} + {workspace.displaySetupWizard ? ( +
  • + + + + + + +
    • + ) : null}
  • diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersForm.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersForm.tsx index f45440d12ab8..8af55ae4e9b0 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersForm.tsx @@ -2,7 +2,6 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import { Field, FieldProps, Form, Formik } from "formik"; -import * as yup from "yup"; import { Input, ControlLabels, DropDown, Button } from "components"; import { Values } from "./types"; @@ -27,21 +26,14 @@ const RequestButton = styled(Button)` min-width: 105px; `; -type ConnectorFormProps = { +type Props = { onSubmit: (values: Values) => void; onCancel: () => void; currentValues?: Values; hasFeedback?: boolean; }; -const requestConnectorValidationSchema = yup.object().shape({ - connectorType: yup.string().required("form.empty.error"), - name: yup.string().required("form.empty.error"), - website: yup.string().required("form.empty.error"), - email: yup.string().email("form.email.error").required("form.empty.error"), -}); - -const ConnectorForm: React.FC = ({ +const InviteUsersForm: React.FC = ({ onSubmit, onCancel, currentValues, @@ -64,9 +56,6 @@ const ConnectorForm: React.FC = ({ website: currentValues?.website || "", email: currentValues?.email || "", }} - validateOnBlur={true} - validateOnChange={true} - validationSchema={requestConnectorValidationSchema} onSubmit={onSubmit} > {({ setFieldValue }) => ( @@ -178,4 +167,4 @@ const ConnectorForm: React.FC = ({ ); }; -export default ConnectorForm; +export default InviteUsersForm; diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx index 40dbbb562804..99c922434ee5 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx @@ -2,12 +2,25 @@ import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import { Field, FieldArray, FieldProps, Form, Formik } from "formik"; +import * as yup from "yup"; import { Button, DropDown, H5, Input, LoadingButton, Modal } from "components"; import { Cell, Header, Row } from "components/SimpleTableComponents"; import { useGetUserService } from "packages/cloud/services/users/UserService"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; +const requestConnectorValidationSchema = yup.object({ + users: yup.array().of( + yup.object().shape({ + role: yup.string().required("form.empty.error"), + email: yup + .string() + .required("form.empty.error") + .email("form.email.error"), + }) + ), +}); + const Content = styled.div` width: 614px; padding: 20px 18px 37px 22px; @@ -31,7 +44,10 @@ const FormRow = styled(Row)` margin-bottom: 8px; `; -export const InviteUsersModal: React.FC<{ onClose: () => void }> = (props) => { +export const InviteUsersModal: React.FC<{ + onSubmit: () => void; + onClose: () => void; +}> = (props) => { const formatMessage = useIntl().formatMessage; const userService = useGetUserService(); const { workspaceId } = useCurrentWorkspace(); @@ -47,88 +63,110 @@ export const InviteUsersModal: React.FC<{ onClose: () => void }> = (props) => { onClose={props.onClose} > { await userService.invite(values.users, workspaceId); + props.onSubmit(); props.onClose(); }} > - {({ values, isValid, isSubmitting, dirty }) => ( -
    - - - -
    - -
    -     - -
    - -
    -     -     - ( - <> - {values.users?.map((_, index) => ( - - - - {({ field }: FieldProps) => ( - - )} - - - - - - - ))} - - - )} - /> + {({ values, isValid, isSubmitting, dirty }) => { + return ( + + + + +
    + +
    +     + +
    + +
    +     +     + ( + <> + {values.users?.map((_, index) => ( + + + + {({ field }: FieldProps) => ( + + )} + + + + + {({ field }: FieldProps) => { + return ( + + ); + }} + + + + ))} + + + )} + /> - - - - - - -     - - )} + + + + + + +     + + ); + }}     ); diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx index c22eb4e60720..d0e9d184dd5d 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx @@ -10,6 +10,9 @@ import Table from "components/Table"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { useGetUserService } from "packages/cloud/services/users/UserService"; import { InviteUsersModal } from "packages/cloud/views/users/InviteUsersModal"; +import { useAuthService } from "packages/cloud/services/auth/AuthService"; +import { useUserHook } from "packages/cloud/services/users/UseUserHook"; +import { User } from "packages/cloud/lib/domain/users"; const Header = styled.div` display: flex; @@ -18,15 +21,20 @@ const Header = styled.div` `; export const UsersSettingsView: React.FC = () => { + const [modalIsOpen, toggleModal] = useToggle(false); const userService = useGetUserService(); const { workspaceId } = useCurrentWorkspace(); - const { data } = useQuery( - ["users"], + + const { data, refetch } = useQuery( + ["users", workspaceId], () => userService.listByWorkspaceId(workspaceId), { suspense: true } ); - const [modalIsOpen, toggleModal] = useToggle(false); + // TODO: show error with request fails + const { isLoading, mutate } = useUserHook(console.log, console.log); + + const { user } = useAuthService(); const columns = React.useMemo( () => [ @@ -52,17 +60,23 @@ export const UsersSettingsView: React.FC = () => { Header: , headerHighlighted: true, accessor: "status", - Cell: (_: CellProps) => + Cell: ({ row }: CellProps) => [ - , // cell.value === "invited" && , ].filter(Boolean), }, ], - [] + [isLoading, mutate] ); + return ( <>
    @@ -74,7 +88,9 @@ export const UsersSettingsView: React.FC = () => {
    • - {modalIsOpen && } + {modalIsOpen && ( + refetch()} /> + )} ); }; diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx index 234ecd7850d6..b6c01f92dc13 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx @@ -43,7 +43,11 @@ const DestinationForm: React.FC = ({ const { destinationDefinitionSpecification, isLoading, + sourceDefinitionError, } = useDestinationDefinitionSpecificationLoad(destinationDefinitionId); + + console.log(sourceDefinitionError); + const onDropDownSelect = (destinationDefinitionId: string) => { setDestinationDefinitionId(destinationDefinitionId); const connector = destinationDefinitions.find( @@ -78,6 +82,7 @@ const DestinationForm: React.FC = ({ }> ` +const Content = styled.div<{ big?: boolean; medium?: boolean }>` width: 100%; - max-width: ${({ big }) => (big ? 1140 : 813)}px; + max-width: ${({ big, medium }) => (big ? 1140 : medium ? 730 : 550)}px; margin: 0 auto; - padding: 33px 0 13px; + padding: 75px 0 30px; display: flex; flex-direction: column; - justify-content: space-between; align-items: center; min-height: 100%; - overflow: hidden; + position: relative; + z-index: 2; `; - -const Main = styled.div` +const ScreenContent = styled.div` width: 100%; -`; - -const Img = styled.img` - text-align: center; - width: 100%; -`; - -const MainTitle = styled(H2)` - margin-top: -39px; - font-family: ${({ theme }) => theme.highlightFont}; - color: ${({ theme }) => theme.darkPrimaryColor}; - letter-spacing: 0.008em; - font-weight: bold; -`; - -const Subtitle = styled.div` - font-size: 14px; - line-height: 21px; - color: ${({ theme }) => theme.greyColor40}; - text-align: center; - margin-top: 7px; -`; - -const StepsCover = styled.div` - margin: 33px 0 28px; -`; - -const TutorialLink = styled(Link)` - margin-top: 32px; - font-size: 14px; - text-align: center; - display: block; -`; - -const PlayIcon = styled(FontAwesomeIcon)` - margin-right: 6px; + position: relative; `; const OnboardingPage: React.FC = () => { const analyticsService = useAnalytics(); - const config = useConfig(); + const { push } = useRouterHook(); useEffect(() => { analyticsService.page("Onboarding Page"); @@ -91,7 +56,8 @@ const OnboardingPage: React.FC = () => { const { sources } = useSourceList(); const { destinations } = useDestinationList(); - + const { connections } = useConnectionList(); + const { syncConnection } = useConnection(); const { sourceDefinitions } = useResource( SourceDefinitionResource.listShape(), {} @@ -103,6 +69,7 @@ const OnboardingPage: React.FC = () => { const { createSource, recreateSource } = useSource(); const { createDestination, recreateDestination } = useDestination(); + const { finishOnboarding } = useWorkspace(); const [successRequest, setSuccessRequest] = useState(false); const [errorStatusRequest, setErrorStatusRequest] = useState<{ @@ -119,6 +86,7 @@ const OnboardingPage: React.FC = () => { const { currentStep, setCurrentStep, steps } = useGetStepsConfig( !!sources.length, !!destinations.length, + !!connections.length, afterUpdateStep ); @@ -129,6 +97,11 @@ const OnboardingPage: React.FC = () => { destinationDefinitions.find((item) => item.destinationDefinitionId === id); const renderStep = () => { + if (currentStep === StepType.INSTRUCTION) { + const onStart = () => setCurrentStep(StepType.CREATE_SOURCE); + //TODO: add username + return ; + } if (currentStep === StepType.CREATE_SOURCE) { const onSubmitSourceStep = async (values: { name: string; @@ -212,7 +185,6 @@ const OnboardingPage: React.FC = () => { availableServices={destinationDefinitions} hasSuccess={successRequest} error={errorStatusRequest} - currentSourceDefinitionId={sources[0].sourceDefinitionId} // destination={ // destinations.length && !successRequest ? destinations[0] : undefined // } @@ -220,42 +192,51 @@ const OnboardingPage: React.FC = () => { ); } + if (currentStep === StepType.SET_UP_CONNECTION) { + return ( + setCurrentStep(StepType.FINAl)} + /> + ); + } + + const onSync = () => syncConnection(connections[0]); + const onCloseOnboarding = () => { + finishOnboarding(); + push(Routes.Root); + }; + return ( - ); }; return ( - - -
    - - - - - - - - - - - {renderStep()} - - - - -
    - -     + + {currentStep === StepType.CREATE_SOURCE ? ( + + ) : currentStep === StepType.CREATE_DESTINATION ? ( + + ) : null} + + + + + }>{renderStep()} + + ); }; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx index bc2549664b48..57b6e4641474 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx @@ -3,28 +3,45 @@ import React from "react"; import CreateConnectionContent from "components/CreateConnectionContent"; import { Source } from "core/resources/Source"; import { Destination } from "core/resources/Destination"; -import { Routes } from "../../routes"; -import useRouter from "hooks/useRouter"; -import SkipOnboardingButton from "./SkipOnboardingButton"; +import TitlesBlock from "./TitlesBlock"; +import { FormattedMessage } from "react-intl"; +import HighlightedText from "./HighlightedText"; type IProps = { errorStatus?: number; source: Source; destination: Destination; + afterSubmitConnection: () => void; }; -const ConnectionStep: React.FC = ({ source, destination }) => { - const { push } = useRouter(); - - const afterSubmitConnection = () => push(Routes.Root); - +const ConnectionStep: React.FC = ({ + source, + destination, + afterSubmitConnection, +}) => { return ( - } - source={source} - destination={destination} - afterSubmitConnection={afterSubmitConnection} - /> + <> + ( + {name} + ), + }} + /> + } + > + + + + ); }; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/DestinationStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/DestinationStep.tsx index d2b217f86066..1b10858137e5 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/DestinationStep.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/DestinationStep.tsx @@ -1,25 +1,22 @@ import React, { useState } from "react"; import { FormattedMessage } from "react-intl"; -import { useResource } from "rest-hooks"; import ContentCard from "components/ContentCard"; import ServiceForm from "views/Connector/ServiceForm"; -import ConnectionBlock from "components/ConnectionBlock"; import { JobsLogItem } from "components/JobItem"; -import SourceDefinitionResource from "core/resources/SourceDefinition"; import { useDestinationDefinitionSpecificationLoad } from "hooks/services/useDestinationHook"; import { createFormErrorMessage } from "utils/errorStatusMessage"; import { JobInfo } from "core/resources/Scheduler"; import { ConnectionConfiguration } from "core/domain/connection"; import { DestinationDefinition } from "core/resources/DestinationDefinition"; -import SkipOnboardingButton from "./SkipOnboardingButton"; +import TitlesBlock from "./TitlesBlock"; +import HighlightedText from "./HighlightedText"; import { useAnalytics } from "hooks/useAnalytics"; type IProps = { availableServices: DestinationDefinition[]; - currentSourceDefinitionId: string; onSubmit: (values: { name: string; serviceType: string; @@ -35,7 +32,6 @@ type IProps = { const DestinationStep: React.FC = ({ onSubmit, availableServices, - currentSourceDefinitionId, hasSuccess, error, jobInfo, @@ -46,9 +42,7 @@ const DestinationStep: React.FC = ({ destinationDefinitionSpecification, isLoading, } = useDestinationDefinitionSpecificationLoad(destinationDefinitionId); - const currentSource = useResource(SourceDefinitionResource.detailShape(), { - sourceDefinitionId: currentSourceDefinitionId, - }); + const analyticsService = useAnalytics(); const onDropDownSelect = (destinationDefinition: string) => { @@ -83,17 +77,23 @@ const DestinationStep: React.FC = ({ return ( <> - - } + ( + {name} + ), + }} + /> + } > + + + - } allowChangeConnector onServiceSelect={onDropDownSelect} onSubmit={onSubmitForm} diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx new file mode 100644 index 000000000000..0261bde007df --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/FinalStep.tsx @@ -0,0 +1,136 @@ +import React, { useEffect, useState } from "react"; +import styled from "styled-components"; +import { FormattedMessage } from "react-intl"; +import { useResource, useSubscription } from "rest-hooks"; + +import VideoItem from "./VideoItem"; +import ProgressBlock from "./ProgressBlock"; +import HighlightedText from "./HighlightedText"; +import { H1, Button } from "components/base"; +import UseCaseBlock from "./UseCaseBlock"; +import ConnectionResource from "core/resources/Connection"; +import SyncCompletedModal from "views/Feedback/SyncCompletedModal"; +import { useOnboardingService } from "hooks/services/Onboarding/OnboardingService"; +import Status from "core/statuses"; +import useWorkspace from "hooks/services/useWorkspace"; + +type FinalStepProps = { + connectionId: string; + onSync: () => void; + onFinishOnboarding: () => void; +}; + +const Title = styled(H1)` + margin: 21px 0; +`; + +const Videos = styled.div` + width: 425px; + height: 205px; + display: flex; + justify-content: space-between; + align-items: center; + margin: 20px 0 50px; + background: url("/video-background.svg") no-repeat; + padding: 0 27px; +`; + +const CloseButton = styled(Button)` + margin-top: 30px; +`; + +const FinalStep: React.FC = ({ + connectionId, + onSync, + onFinishOnboarding, +}) => { + const { sendFeedback } = useWorkspace(); + const { + feedbackPassed, + passFeedback, + useCases, + skipCase, + } = useOnboardingService(); + const connection = useResource(ConnectionResource.detailShape(), { + connectionId, + }); + useSubscription(ConnectionResource.detailShape(), { + connectionId: connectionId, + }); + const [isOpen, setIsOpen] = useState(false); + + useEffect(() => { + if ( + connection.latestSyncJobStatus === Status.SUCCEEDED && + !feedbackPassed + ) { + setIsOpen(true); + } + }, [connection.latestSyncJobStatus, feedbackPassed]); + + const onSendFeedback = (feedback: string) => { + sendFeedback({ + feedback, + source: connection.source, + destination: connection.destination, + }); + passFeedback(); + setIsOpen(false); + }; + + return ( + <> + + } + videoId="sKDviQrOAbU" + img="/videoCover.png" + /> + } + videoId="sKDviQrOAbU" + img="/videoCover.png" + /> + + {!feedbackPassed && ( + + )} + + + <FormattedMessage + id="onboarding.useCases" + values={{ + name: (...name: React.ReactNode[]) => ( + <HighlightedText>{name}</HighlightedText> + ), + }} + /> + + + {useCases && + useCases.map((item, key) => ( + + ))} + + + + + + {isOpen ? ( + setIsOpen(false)} + onPassFeedback={onSendFeedback} + /> + ) : null} + + ); +}; + +export default FinalStep; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/HighlightedText.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/HighlightedText.tsx new file mode 100644 index 000000000000..c998c6254cd6 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/HighlightedText.tsx @@ -0,0 +1,7 @@ +import styled from "styled-components"; + +const HighlightedText = styled.span` + color: ${({ theme }) => theme.redColor}; +`; + +export default HighlightedText; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/LetterLine.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/LetterLine.tsx new file mode 100644 index 000000000000..c0a7f05ce532 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/LetterLine.tsx @@ -0,0 +1,86 @@ +import React from "react"; +import styled, { keyframes } from "styled-components"; + +export const RollAnimation = keyframes` + 0% { + width: 0; + } + 100% { + width: 100%; + } +`; + +export const ExitRollAnimation = keyframes` + 0% { + width: 100%; + float: right; + } + 100% { + width: 0; + float: right; + } +`; + +export const EnterAnimation = keyframes` + 0% { + left: -78px; + } + 100% { + left: calc(50% - 39px); + } +`; + +export const ExitAnimation = keyframes` + 0% { + left: calc(50% - 39px); + } + 100% { + left: calc(100% + 78px); + } +`; + +const Line = styled.div<{ onRight?: boolean }>` + position: absolute; + width: calc(50% - 275px); + z-index: 1; + top: 382px; + left: ${({ onRight }) => (onRight ? "calc(50% + 275px)" : 0)}; +`; +const Path = styled.div<{ exit?: boolean }>` + width: 100%; + height: 2px; + background: ${({ theme }) => theme.primaryColor}; + animation: ${({ exit }) => (exit ? ExitRollAnimation : RollAnimation)} 0.6s + linear ${({ exit }) => (exit ? 0.8 : 0)}s; + animation-fill-mode: forwards; +`; +const Img = styled.img<{ exit?: boolean }>` + position: absolute; + top: -58px; + left: -78px; + animation: ${({ exit }) => (exit ? ExitAnimation : EnterAnimation)} 0.8s + linear ${({ exit }) => (exit ? 0 : 0.6)}s; + animation-fill-mode: both; +`; + +type LetterLineProps = { + onRight?: boolean; + exit?: boolean; +}; + +const LetterLine: React.FC = ({ onRight, exit }) => { + return ( + + + newsletter + + ); +}; + +export default LetterLine; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx new file mode 100644 index 000000000000..302441becb86 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx @@ -0,0 +1,127 @@ +import React from "react"; +import { FormattedMessage } from "react-intl"; +import styled, { keyframes } from "styled-components"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faChevronRight } from "@fortawesome/free-solid-svg-icons"; + +import { Connection } from "core/domain/connection"; +import Link from "components/Link"; +import { Button, H1 } from "components/base"; +import { Routes } from "pages/routes"; +import Status from "core/statuses"; + +const run = keyframes` + from { + background-position: 0 0; + } + + to { + background-position: 98% 0; + } +`; + +const Bar = styled.div` + width: 100%; + height: 49px; + background: ${({ theme }) => theme.darkBeigeColor} url("/rectangle.svg"); + color: ${({ theme }) => theme.redColor}; + border-radius: 15px; + font-weight: 500; + font-size: 13px; + line-height: 16px; + display: flex; + justify-content: center; + align-items: center; + + animation: ${run} 15s linear infinite; +`; +const Lnk = styled(Link)` + font-weight: 600; + text-decoration: underline; + color: ${({ theme }) => theme.redColor}; + padding: 0 5px; +`; +const Img = styled.img` + margin-right: 9px; +`; +const ControlBlock = styled.div` + height: 49px; + text-align: center; + display: flex; + justify-content: center; + align-items: center; +`; +const PaddedButton = styled(Button)` + margin-left: 10px; +`; + +type ProgressBlockProps = { + connection: Connection; + onSync: () => void; +}; + +const ProgressBlock: React.FC = ({ + connection, + onSync, +}) => { + const showMessage = (status: string | null) => { + if (status === null || !status) { + return ; + } + if (status === Status.FAILED) { + return ; + } + if (status === Status.CANCELLED) { + return ; + } + + return ""; + }; + + if (connection.latestSyncJobStatus === Status.SUCCEEDED) { + return null; + } + + if ( + connection.latestSyncJobStatus !== Status.RUNNING && + connection.latestSyncJobStatus !== Status.INCOMPLETE + ) { + return ( + +

    {showMessage(connection.latestSyncJobStatus)}

    + + + +     + ); + } + + return ( + + + ( + <> + {sr}{" "} + + + ), + ds: (...ds: React.ReactNode[]) => ( + + {ds} + + ), + sync: (...sync: React.ReactNode[]) => ( + + {sync} + + ), + }} + /> + + ); +}; + +export default ProgressBlock; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/SourceStep.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/SourceStep.tsx index dadcfd29167f..8b9bd582a81b 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/SourceStep.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/SourceStep.tsx @@ -11,9 +11,10 @@ import { JobsLogItem } from "components/JobItem"; import { useSourceDefinitionSpecificationLoad } from "hooks/services/useSourceHook"; -import SkipOnboardingButton from "./SkipOnboardingButton"; import { createFormErrorMessage } from "utils/errorStatusMessage"; import { useAnalytics } from "hooks/useAnalytics"; +import HighlightedText from "./HighlightedText"; +import TitlesBlock from "./TitlesBlock"; type IProps = { onSubmit: (values: { @@ -72,24 +73,39 @@ const SourceStep: React.FC = ({ const errorMessage = error ? createFormErrorMessage(error) : ""; return ( - }> - + <> + ( + {name} + ), + }} + /> } - allowChangeConnector - onServiceSelect={onServiceSelect} - onSubmit={onSubmitForm} - formType="source" - availableServices={availableServices} - hasSuccess={hasSuccess} - errorMessage={errorMessage} - specifications={sourceDefinitionSpecification?.connectionSpecification} - documentationUrl={sourceDefinitionSpecification?.documentationUrl} - isLoading={isLoading} - /> - - + > + + + + + + + ); }; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/StepsCounter.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/StepsCounter.tsx new file mode 100644 index 000000000000..9d954cc86db9 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/StepsCounter.tsx @@ -0,0 +1,71 @@ +import React from "react"; +import styled from "styled-components"; + +import StepItem from "./components/StepItem"; +import StarsIcon from "./components/StarsIcon"; +import { StepType } from "../../types"; + +type StepsCounterProps = { + steps: { id: StepType; name?: React.ReactNode }[]; + currentStep: StepType; +}; + +const Steps = styled.div` + display: flex; + flex-direction: row; +`; + +const Content = styled.div` + position: relative; + display: flex; + flex-direction: row; +`; + +const Rocket = styled.img<{ stepNumber: number }>` + position: absolute; + width: 87px; + transform: matrix(0.99, 0.12, -0.12, 0.99, 0, 0) rotate(6.73deg); + top: 1px; + left: ${({ stepNumber }) => -23 + stepNumber * 95.5}px; + transition: 0.8s; +`; + +const Stars = styled.div<{ isLastStep?: boolean }>` + position: absolute; + top: -23px; + right: -35px; + color: ${({ theme }) => theme.dangerColor}; + opacity: ${({ isLastStep }) => (isLastStep ? 1 : 0)}; + transition: 0.8s 0.2s; +`; + +const StepsCounter: React.FC = ({ steps, currentStep }) => { + const stepItem = steps.find((item) => item.id === currentStep); + const stepIndex = stepItem ? steps.indexOf(stepItem) : 0; + const isLastStep = currentStep === steps[steps.length - 1].id; + + return ( + + + {steps.map((stepItem, key) => ( + = key} + current={stepItem.id === currentStep} + > + {key === steps.length - 1 ? : key} + + ))} + + + + + + + ); +}; + +export default StepsCounter; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/components/StarsIcon.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/components/StarsIcon.tsx new file mode 100644 index 000000000000..5e7ee80d0f3f --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/components/StarsIcon.tsx @@ -0,0 +1,22 @@ +const StarsIcon = ({ + color = "currentColor", +}: { + color?: string; +}): JSX.Element => ( + + + + + +); + +export default StarsIcon; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/components/StepItem.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/components/StepItem.tsx new file mode 100644 index 000000000000..bc71b20ed6c9 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/components/StepItem.tsx @@ -0,0 +1,74 @@ +import React from "react"; +import styled from "styled-components"; + +type StepItemProps = { + active?: boolean; + current?: boolean; + children?: React.ReactNode; +}; + +const Content = styled.div<{ active?: boolean }>` + display: flex; + flex-direction: row; + align-items: center; + + &:last-child > .next-path { + display: none; + } + + &:first-child > .previous-path { + display: none; + } +`; + +const Item = styled.div<{ active?: boolean }>` + height: 46px; + width: 46px; + border-radius: 50%; + padding: 6px 5px; + border: 1px solid + ${({ theme, active }) => + active ? theme.primaryColor : theme.lightTextColor}; + background: ${({ theme, active }) => + active ? theme.primaryColor : theme.transparentColor}; + color: ${({ theme, active }) => + active ? theme.whiteColor : theme.lightTextColor}; + font-weight: normal; + font-size: 18px; + line-height: 22px; + display: flex; + justify-content: center; + align-items: center; + transition: 0.8s; +`; + +const Path = styled.div<{ active?: boolean }>` + width: 25px; + height: 1px; + background: ${({ theme }) => theme.lightTextColor}; + + &:before { + content: ""; + display: block; + width: ${({ active }) => (active ? 25 : 0)}px; + height: 1px; + background: ${({ theme }) => theme.primaryColor}; + transition: 0.8s 0.5s; + } + + &:first-child:before { + transition: 0.8s; + } +`; + +const StepItem: React.FC = ({ active, children }) => { + return ( + + + {children} + + + ); +}; + +export default StepItem; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/index.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/index.tsx new file mode 100644 index 000000000000..de9748ebc946 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/StepsCounter/index.tsx @@ -0,0 +1,4 @@ +import StepsCounter from "./StepsCounter"; + +export default StepsCounter; +export { StepsCounter }; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx new file mode 100644 index 000000000000..3b2480264f4f --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/TitlesBlock.tsx @@ -0,0 +1,35 @@ +import React from "react"; +import { H1 } from "components/base"; +import styled from "styled-components"; + +type TitlesBlockProps = { + title: React.ReactNode; + children?: React.ReactNode; +}; + +const TitlesContent = styled.div` + padding: 42px 0 33px; + color: ${({ theme }) => theme.textColor}; + max-width: 493px; +`; + +const Text = styled.div` + padding-top: 10px; + font-weight: normal; + font-size: 13px; + line-height: 20px; + text-align: center; +`; + +const TitlesBlock: React.FC = ({ title, children }) => { + return ( + +

    + {title} +

    + {children} +     + ); +}; + +export default TitlesBlock; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx new file mode 100644 index 000000000000..8b4de0097143 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/UseCaseBlock.tsx @@ -0,0 +1,60 @@ +import React from "react"; +import styled from "styled-components"; + +import ContentCard from "components/ContentCard"; +import { FormattedMessage } from "react-intl"; + +type UseCaseBlockProps = { + count: number; + id: string; + onSkip: (id: string) => void; +}; + +const Block = styled(ContentCard)` + margin-bottom: 10px; + width: 100%; + padding: 16px; + display: flex; + justify-content: space-between; + flex-direction: row; + align-items: center; + font-size: 16px; + line-height: 28px; +`; + +const Num = styled.div` + width: 28px; + height: 28px; + border-radius: 50%; + background: ${({ theme }) => theme.primaryColor}; + color: ${({ theme }) => theme.whiteColor}; + margin-right: 13px; + font-weight: bold; + font-size: 12px; + line-height: 28px; + display: inline-block; + text-align: center; +`; + +const SkipButton = styled.div` + color: ${({ theme }) => theme.lightTextColor}; + font-size: 16px; + line-height: 28px; + cursor: pointer; +`; + +const UseCaseBlock: React.FC = ({ id, count, onSkip }) => { + return ( + +
    + {count} + +
    + onSkip(id)}> + + +     + ); +}; + +export default UseCaseBlock; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/VideoItem.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/VideoItem.tsx new file mode 100644 index 000000000000..a112c1de191b --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/VideoItem.tsx @@ -0,0 +1,108 @@ +import React, { useState } from "react"; +import styled from "styled-components"; + +import ShowVideo from "./components/ShowVideo"; +import PlayButton from "./components/PlayButton"; + +type VideoItemProps = { + small?: boolean; + videoId?: string; + img?: string; + description?: React.ReactNode; +}; + +const Content = styled.div<{ small?: boolean }>` + width: ${({ small }) => (small ? 158 : 317)}px; +`; + +const VideoBlock = styled.div<{ small?: boolean }>` + position: relative; + width: 100%; + height: ${({ small }) => (small ? 92 : 185)}px; + filter: drop-shadow(0px 14.4px 14.4px rgba(26, 25, 77, 0.2)); + + &:before, + &:after { + content: ""; + display: block; + position: absolute; + top: 0; + left: 0; + border-radius: ${({ small }) => (small ? 3.6 : 7.2)}px; + } + + &:before { + width: ${({ small }) => (small ? 158 : 317)}px; + height: ${({ small }) => (small ? 94 : 189)}px; + transform: rotate(2.98deg); + background: ${({ theme }) => theme.primaryColor}; + z-index: 1; + } + + &:after { + width: ${({ small }) => (small ? 160 : 320)}px; + height: ${({ small }) => (small ? 92 : 184)}px; + transform: rotate(-2.48deg); + background: ${({ theme }) => theme.successColor}; + z-index: 2; + } +`; + +const VideoFrame = styled.div<{ small?: boolean; img?: string }>` + cursor: pointer; + position: relative; + width: ${({ small }) => (small ? 158 : 317)}px; + height: ${({ small }) => (small ? 92 : 185)}px; + background: ${({ theme }) => theme.whiteColor} + ${({ img }) => (img ? `url(${img})` : "")}; + background-size: cover; + border: 2.4px solid ${({ theme }) => theme.whiteColor}; + box-shadow: 0 2.4px 4.8px rgba(26, 25, 77, 0.12), + 0 16.2px 7.2px -10.2px rgba(26, 25, 77, 0.2); + border-radius: ${({ small }) => (small ? 3.6 : 7.2)}px; + z-index: 3; + display: flex; + justify-content: center; + align-items: center; +`; + +const Description = styled.div<{ small?: boolean }>` + text-align: center; + color: ${({ theme, small }) => + small ? theme.textColor : theme.primaryColor}; + font-size: 13px; + line-height: ${({ small }) => (small ? 16 : 20)}px; + margin-top: 14px; + cursor: pointer; +`; + +const VideoItem: React.FC = ({ + description, + small, + videoId, + img, +}) => { + const [isVideoOpen, setIsVideoOpen] = useState(false); + + return ( + + + setIsVideoOpen(true)} + > + setIsVideoOpen(true)} /> + + + setIsVideoOpen(true)}> + {description} + + {isVideoOpen ? ( + setIsVideoOpen(false)} /> + ) : null} + + ); +}; + +export default VideoItem; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/PlayButton.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/PlayButton.tsx new file mode 100644 index 000000000000..4770d5464cb2 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/PlayButton.tsx @@ -0,0 +1,103 @@ +import React from "react"; +import styled, { keyframes } from "styled-components"; + +type PlayButtonProps = { + small?: boolean; + onClick: () => void; +}; + +export const BigCircleAnimation = keyframes` + 0% { + height: 80%; + width: 80%; + } + 100% { + width: 100%; + height: 100%; + } +`; + +export const MiddleCircleAnimation = keyframes` + 0% { + height: 53%; + width: 53%; + } + 100% { + width: 73%; + height: 73%; + } +`; + +export const SmallCircleAnimation = keyframes` + 0% { + height: 20%; + width: 20%; + } + 100% { + width: 40%; + height: 40%; + } +`; + +const MainCircle = styled.div` + cursor: pointer; + height: ${({ small }) => (small ? 42 : 85)}px; + width: ${({ small }) => (small ? 42 : 85)}px; + border-radius: 50%; + background: ${({ theme }) => theme.primaryColor}; + padding: ${({ small }) => (small ? "10px 0 0 16px" : "20px 0 0 32px")}; + box-shadow: 0 2.4px 4.8px ${({ theme }) => theme.cardShadowColor}, + 0 16.2px 7.2px -10.2px ${({ theme }) => theme.cardShadowColor}; + + &:hover { + display: flex; + justify-content: center; + align-items: center; + padding: 0; + + & > img { + display: none; + } + & div { + display: flex; + justify-content: center; + align-items: center; + } + } +`; + +const BigCircle = styled.div<{ small?: boolean }>` + height: ${({ small }) => (small ? 32 : 65)}px; + width: ${({ small }) => (small ? 32 : 65)}px; + border-radius: 50%; + background: rgba(255, 255, 255, 0.5); + display: none; + animation: ${BigCircleAnimation} alternate 0.5s linear 0s infinite; +`; + +const MiddleCircle = styled(BigCircle)` + height: ${({ small }) => (small ? 22 : 45)}px; + width: ${({ small }) => (small ? 22 : 45)}px; + animation-name: ${MiddleCircleAnimation}; +`; + +const SmallCircle = styled(BigCircle)` + height: ${({ small }) => (small ? 8 : 17)}px; + width: ${({ small }) => (small ? 8 : 17)}px; + animation-name: ${SmallCircleAnimation}; +`; + +const PlayButton: React.FC = ({ small, onClick }) => { + return ( + + play + + + + + + + ); +}; + +export default PlayButton; diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx new file mode 100644 index 000000000000..151ea3fbef03 --- /dev/null +++ b/airbyte-webapp/src/pages/OnboardingPage/components/VideoItem/components/ShowVideo.tsx @@ -0,0 +1,45 @@ +import React from "react"; +import styled from "styled-components"; +import { faTimes } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; + +import Modal from "components/Modal"; +import { Button } from "components/base"; + +type ShowVideoProps = { + videoId?: string; + onClose: () => void; +}; + +const CloseButton = styled(Button)` + position: absolute; + top: 30px; + right: 30px; + color: ${({ theme }) => theme.whiteColor}; + font-size: 20px; + + &:hover { + border: none; + } +`; + +const ShowVideo: React.FC = ({ videoId, onClose }) => { + return ( + + + + +