From afde39e819ca7dcc783ca5d5fdf0a71a47c06400 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Fri, 28 Oct 2022 07:21:24 -0500 Subject: [PATCH 01/10] add boilerplate --- .../source-partnerstack/.dockerignore | 6 ++ .../connectors/source-partnerstack/Dockerfile | 38 +++++++ .../connectors/source-partnerstack/README.md | 79 +++++++++++++++ .../source-partnerstack/__init__.py | 3 + .../acceptance-test-config.yml | 25 +++++ .../acceptance-test-docker.sh | 16 +++ .../source-partnerstack/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 16 +++ .../integration_tests/configured_catalog.json | 67 +++++++++++++ .../integration_tests/invalid_config.json | 5 + .../integration_tests/sample_config.json | 4 + .../connectors/source-partnerstack/main.py | 13 +++ .../source-partnerstack/requirements.txt | 2 + .../connectors/source-partnerstack/setup.py | 29 ++++++ .../source_partnerstack/__init__.py | 8 ++ .../source_partnerstack/partnerstack.yaml | 98 +++++++++++++++++++ .../source_partnerstack/source.py | 18 ++++ .../source_partnerstack/spec.yaml | 27 +++++ 19 files changed, 466 insertions(+) create mode 100644 airbyte-integrations/connectors/source-partnerstack/.dockerignore create mode 100644 airbyte-integrations/connectors/source-partnerstack/Dockerfile create mode 100644 airbyte-integrations/connectors/source-partnerstack/README.md create mode 100644 airbyte-integrations/connectors/source-partnerstack/__init__.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-partnerstack/build.gradle create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/main.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/requirements.txt create mode 100644 airbyte-integrations/connectors/source-partnerstack/setup.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/__init__.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml diff --git a/airbyte-integrations/connectors/source-partnerstack/.dockerignore b/airbyte-integrations/connectors/source-partnerstack/.dockerignore new file mode 100644 index 000000000000..92734c506d7d --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_partnerstack +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-partnerstack/Dockerfile b/airbyte-integrations/connectors/source-partnerstack/Dockerfile new file mode 100644 index 000000000000..3c43983b269d --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_partnerstack ./source_partnerstack + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-partnerstack diff --git a/airbyte-integrations/connectors/source-partnerstack/README.md b/airbyte-integrations/connectors/source-partnerstack/README.md new file mode 100644 index 000000000000..c2d61c9435fd --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/README.md @@ -0,0 +1,79 @@ +# Partnerstack Source + +This is the repository for the Partnerstack configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/partnerstack). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-partnerstack:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/partnerstack) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_partnerstack/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source partnerstack test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-partnerstack:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-partnerstack:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-partnerstack:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-partnerstack:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-partnerstack:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-partnerstack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-partnerstack:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-partnerstack:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-partnerstack/__init__.py b/airbyte-integrations/connectors/source-partnerstack/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml new file mode 100644 index 000000000000..ed96b56e460d --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml @@ -0,0 +1,25 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-partnerstack:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_partnerstack/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [deals] + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-partnerstack/build.gradle b/airbyte-integrations/connectors/source-partnerstack/build.gradle new file mode 100644 index 000000000000..61d7a10468a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_partnerstack' +} diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/__init__.py b/airbyte-integrations/connectors/source-partnerstack/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..f86aa59251d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/configured_catalog.json @@ -0,0 +1,67 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "leads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "partnerships", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "rewards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/invalid_config.json new file mode 100644 index 000000000000..ec176d99961a --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "public_key": "", + "private_key": "", + "start_date": "2022-10-25" +} diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_config.json new file mode 100644 index 000000000000..9b24d9bcd386 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "public_key": "", + "private_key": "" +} diff --git a/airbyte-integrations/connectors/source-partnerstack/main.py b/airbyte-integrations/connectors/source-partnerstack/main.py new file mode 100644 index 000000000000..aa7176881f62 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_partnerstack import SourcePartnerstack + +if __name__ == "__main__": + source = SourcePartnerstack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-partnerstack/requirements.txt b/airbyte-integrations/connectors/source-partnerstack/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-partnerstack/setup.py b/airbyte-integrations/connectors/source-partnerstack/setup.py new file mode 100644 index 000000000000..64bade243e1a --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_partnerstack", + description="Source implementation for Partnerstack.", + author="Elliot Trabac", + author_email="elliot.trabac1@gmail.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/__init__.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/__init__.py new file mode 100644 index 000000000000..30492d64a767 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePartnerstack + +__all__ = ["SourcePartnerstack"] diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml new file mode 100644 index 000000000000..fb9a11d36ae5 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml @@ -0,0 +1,98 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: ["data", "items"] + requester: + url_base: "https://api.partnerstack.com/api/v2/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['public_key'] }}" + password: "{{ config['private_key'] }}" + request_options_provider: + request_parameters: + min_created: "{{ timestamp(config['start_date']) * 1000 }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records[-1]['key'] }}" + stop_condition: "{{ response.data.has_more is false }}" + page_size: 250 + page_size_option: + field_name: "limit" + inject_into: "request_parameter" + page_token_option: + field_name: "starting_after" + inject_into: "request_parameter" + url_base: + $ref: "*ref(definitions.requester.url_base)" + requester: + $ref: "*ref(definitions.requester)" + + # base stream + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + + # stream definitions + customers_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "customers" + primary_key: "key" + path: "/customers" + deals_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "deals" + primary_key: "key" + path: "/deals" + groups_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "groups" + primary_key: "key" + path: "/groups" + leads_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "leads" + primary_key: "key" + path: "/leads" + partnerships_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "partnerships" + primary_key: "key" + path: "/partnerships" + rewards_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "rewards" + primary_key: "key" + path: "/rewards" + transactions_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "transactions" + primary_key: "key" + path: "/transactions" + +streams: + - "*ref(definitions.customers_stream)" + - "*ref(definitions.deals_stream)" + - "*ref(definitions.groups_stream)" + - "*ref(definitions.leads_stream)" + - "*ref(definitions.partnerships_stream)" + - "*ref(definitions.rewards_stream)" + - "*ref(definitions.transactions_stream)" + +check: + stream_names: + - "groups" diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py new file mode 100644 index 000000000000..6584be3472c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourcePartnerstack(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "partnerstack.yaml"}) diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml new file mode 100644 index 000000000000..2e884ad034d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml @@ -0,0 +1,27 @@ +documentationUrl: https://docs.partnerstack.com/reference/auth +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Partnerstack Spec + type: object + required: + - public_key + - private_key + additionalProperties: true + properties: + public_key: + type: string + description: The Live Public Key for a Partnerstack account. + airbyte_secret: true + private_key: + type: string + description: The Live Private Key for a Partnerstack account. + airbyte_secret: true + start_date: + type: string + title: Start date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - "2017-01-25T00:00:00Z" From 58ef64080a50a557a0757c03f7a820e0dc81cf85 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Fri, 28 Oct 2022 07:21:41 -0500 Subject: [PATCH 02/10] configure schemas --- .../integration_tests/simple_catalog.json | 13 ++++ .../schemas/customers.json | 51 ++++++++++++++ .../source_partnerstack/schemas/deals.json | 66 +++++++++++++++++++ .../source_partnerstack/schemas/groups.json | 30 +++++++++ .../source_partnerstack/schemas/leads.json | 45 +++++++++++++ .../schemas/partnerships.json | 48 ++++++++++++++ .../source_partnerstack/schemas/rewards.json | 45 +++++++++++++ .../schemas/transactions.json | 42 ++++++++++++ 8 files changed, 340 insertions(+) create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/deals.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/groups.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/leads.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/partnerships.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/rewards.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/transactions.json diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json new file mode 100644 index 000000000000..33ac5d771621 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "leads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/customers.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/customers.json new file mode 100644 index 000000000000..762ece7fa09a --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/customers.json @@ -0,0 +1,51 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "integer"] + }, + "customer_key": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "field_data": { + "type": ["null", "object"] + }, + "fields": { + "type": ["null", "array"] + }, + "key": { + "type": ["null", "string"] + }, + "meta": { + "type": ["null", "object"] + }, + "name": { + "type": ["null", "string"] + }, + "partner_key": { + "type": ["null", "string"] + }, + "partnership_key": { + "type": ["null", "string"] + }, + "provider_key": { + "type": ["null", "string"] + }, + "source_key": { + "type": ["null", "string"] + }, + "source_type": { + "type": ["null", "string"] + }, + "test": { + "type": ["null", "boolean"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/deals.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/deals.json new file mode 100644 index 000000000000..1c5366a328c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/deals.json @@ -0,0 +1,66 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"] + }, + "account_name": { + "type": ["null", "object"] + }, + "amount": { + "type": ["null", "integer"] + }, + "close_date": { + "type": ["null", "string"] + }, + "contact_first_name": { + "type": ["null", "string"] + }, + "contact_last_name": { + "type": ["null", "string"] + }, + "external_key": { + "type": ["null", "string"] + }, + "field_data": { + "type": ["null", "object"] + }, + "fields": { + "type": ["null", "array"] + }, + "group_key": { + "type": ["null", "string"] + }, + "lost_reason": { + "type": ["null", "string"] + }, + "meta": { + "type": ["null", "object"] + }, + "mold_key": { + "type": ["null", "string"] + }, + "partner_key": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "stage": { + "type": ["null", "string"] + }, + "team": { + "type": ["null", "object"] + }, + "team_member": { + "type": ["null", "object"] + } + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/groups.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/groups.json new file mode 100644 index 000000000000..3b92990e307f --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/groups.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "archived": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "features": { + "type": ["null", "object"] + }, + "key": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "slug": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/leads.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/leads.json new file mode 100644 index 000000000000..d917c20a0670 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/leads.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "approved": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "integer"] + }, + "external_key": { + "type": ["null", "string"] + }, + "field_data": { + "type": ["null", "object"] + }, + "fields": { + "type": ["null", "array"] + }, + "group_key": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "meta": { + "type": ["null", "object"] + }, + "mold_key": { + "type": ["null", "string"] + }, + "partner_key": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/partnerships.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/partnerships.json new file mode 100644 index 000000000000..63488aa0d686 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/partnerships.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "group": { + "type": ["null", "object"] + }, + "joined_at": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "manager": { + "type": ["null", "object"] + }, + "m_data": { + "type": ["null", "object"] + }, + "partner_key": { + "type": ["null", "string"] + }, + "stats": { + "type": ["null", "object"] + }, + "tags": { + "type": ["null", "array"] + }, + "team": { + "type": ["null", "object"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/rewards.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/rewards.json new file mode 100644 index 000000000000..5ceb04163596 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/rewards.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "action": { + "type": ["null", "object"] + }, + "amount": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "integer"] + }, + "customer": { + "type": ["null", "object"] + }, + "description": { + "type": ["null", "string"] + }, + "invoice": { + "type": ["null", "object"] + }, + "key": { + "type": ["null", "string"] + }, + "offer": { + "type": ["null", "object"] + }, + "partnership": { + "type": ["null", "object"] + }, + "reward_status": { + "type": ["null", "string"] + }, + "transaction": { + "type": ["null", "object"] + }, + "trigger": { + "type": ["null", "object"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/transactions.json b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/transactions.json new file mode 100644 index 000000000000..8a903c76ce56 --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/schemas/transactions.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_usd": { + "type": ["null", "integer"] + }, + "approved": { + "type": ["null", "boolean"] + }, + "category_key": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"] + }, + "customer_key": { + "type": ["null", "string"] + }, + "extension": { + "type": ["null", "object"] + }, + "key": { + "type": ["null", "string"] + }, + "product_key": { + "type": ["null", "object"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} From 9d2319bc923326d2fd77ffec975dd75b0ffbc89f Mon Sep 17 00:00:00 2001 From: Elliot T Date: Fri, 28 Oct 2022 07:21:57 -0500 Subject: [PATCH 03/10] add connector doc --- docs/integrations/sources/partnerstack.md | 41 +++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/integrations/sources/partnerstack.md diff --git a/docs/integrations/sources/partnerstack.md b/docs/integrations/sources/partnerstack.md new file mode 100644 index 000000000000..fb38192b4324 --- /dev/null +++ b/docs/integrations/sources/partnerstack.md @@ -0,0 +1,41 @@ +# Partnerstack + +## Sync overview + +The Partnerstack source supports both Full Refresh only. + +This source can sync data for the [Partnerstack API](https://docs.partnerstack.com/reference). + +### Output schema + +This Source is capable of syncing the following core Streams: + +- [Customers](https://docs.partnerstack.com/reference/get_v2-customers-2) +- [Deals](https://docs.partnerstack.com/reference/get_v2-deals) +- [Groups](https://docs.partnerstack.com/reference/get_groups) +- [Leads](https://docs.partnerstack.com/reference/get_leads) +- [Partnerships](https://docs.partnerstack.com/reference/get_v2-partnerships-2) +- [Rewards](https://docs.partnerstack.com/reference/get_v2-rewards-2) +- [Transactions](https://docs.partnerstack.com/reference/get_v2-transactions-2) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | No | | +| Namespaces | No | | + +### Performance considerations + +The Partnerstack connector should not run into Partnerstack API limitations under normal usage. + +## Requirements + +- **Partnerstack API keys**. See the [Partnerstack docs](https://docs.partnerstack.com/reference/auth) for information on how to obtain the API keys. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------- | :-------------------------------- | +| 0.1.0 | 2022-10-27 | [XXX](https://github.com/airbytehq/airbyte/pull/XXX) | Add Partnerstack Source Connector | From 263a9f8db6291ae65eed314e7beea5030e09699a Mon Sep 17 00:00:00 2001 From: Elliot T Date: Fri, 28 Oct 2022 07:22:04 -0500 Subject: [PATCH 04/10] update airbyte docs --- airbyte-integrations/builds.md | 1 + docs/integrations/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 28fc978af3bf..bad06c64231c 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -82,6 +82,7 @@ | OpenWeather | [![source-openweather](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-openweather%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-openweather) | | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | | Orbit | [![source-orbit](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-orbit%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-orbit) | +| Partnerstack | [![source-partnerstack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-partnerstack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-partnerstack) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | | Paystack | [![source-paystack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paystack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paystack) | | PersistIq | [![source-persistiq](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-persistiq%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-persistiq/) | diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 578fd492e152..3569bef9aeb6 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -124,6 +124,7 @@ For more information about the grading system, see [Product Release Stages](http | [Orbit](sources/orbit.md) | Alpha | Yes | | [Outreach](./sources/outreach.md) | Alpha | No | | [PagerDuty](sources/pagerduty.md) | Alpha | No | +| [Partnerstack](sources/partnerstack.md) | Alpha | No | | [PayPal Transaction](sources/paypal-transaction.md) | Generally Available | Yes | | [Paystack](sources/paystack.md) | Alpha | No | | [PersistIq](sources/persistiq.md) | Alpha | Yes | From 713a66f57f12095a1b8c7e09ae9f3073edcc1fbe Mon Sep 17 00:00:00 2001 From: Elliot T Date: Sun, 30 Oct 2022 20:49:54 -0400 Subject: [PATCH 05/10] edit spec --- .../source-partnerstack/source_partnerstack/spec.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml index 2e884ad034d7..475a1b42ab69 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docs.partnerstack.com/reference/auth +documentationUrl: https://docs.airbyte.com/integrations/sources/partnerstack connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Partnerstack Spec @@ -10,10 +10,12 @@ connectionSpecification: properties: public_key: type: string + title: Partnerstack Public key description: The Live Public Key for a Partnerstack account. airbyte_secret: true private_key: type: string + title: Partnerstack Private key description: The Live Private Key for a Partnerstack account. airbyte_secret: true start_date: From 69ce25c2318c72ca108458091349fbdc8fe5557d Mon Sep 17 00:00:00 2001 From: Elliot T Date: Mon, 7 Nov 2022 14:32:06 +0100 Subject: [PATCH 06/10] configure incremental sync --- .../integration_tests/abnormal_state.json | 5 +++++ .../integration_tests/sample_state.json | 5 +++++ .../integration_tests/simple_catalog.json | 6 +++--- .../source_partnerstack/partnerstack.yaml | 16 +++++++++++++++- 4 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..919cab1d149d --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "customers": { + "updated_at": "1667643069899" + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json new file mode 100644 index 000000000000..919cab1d149d --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "customers": { + "updated_at": "1667643069899" + } +} diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json index 33ac5d771621..e662b9017a9a 100644 --- a/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json @@ -2,11 +2,11 @@ "streams": [ { "stream": { - "name": "leads", + "name": "customers", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml index fb9a11d36ae5..b4654ba73dc3 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml @@ -13,7 +13,18 @@ definitions: password: "{{ config['private_key'] }}" request_options_provider: request_parameters: - min_created: "{{ timestamp(config['start_date']) * 1000 }}" + min_created: "{{ timestamp(stream_slice['start_time']) * 1000 }}" + stream_slicer: + type: "DatetimeStreamSlicer" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f%z" + step: "30d" + cursor_field: "{{ options.stream_cursor_field }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" retriever: record_selector: $ref: "*ref(definitions.selector)" @@ -34,6 +45,8 @@ definitions: $ref: "*ref(definitions.requester.url_base)" requester: $ref: "*ref(definitions.requester)" + stream_slicer: + $ref: "*ref(definitions.stream_slicer)" # base stream base_stream: @@ -47,6 +60,7 @@ definitions: name: "customers" primary_key: "key" path: "/customers" + stream_cursor_field: "updated_at" deals_stream: $ref: "*ref(definitions.base_stream)" $options: From d3a3d92ca3e1f7107ad71fc44a73b00d0810d92a Mon Sep 17 00:00:00 2001 From: Elliot T Date: Mon, 7 Nov 2022 20:35:15 +0100 Subject: [PATCH 07/10] use custom components --- .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/sample_state.json | 2 +- .../source_partnerstack/components.py | 62 +++++++++++++++++++ .../source_partnerstack/partnerstack.yaml | 15 ++--- 4 files changed, 68 insertions(+), 13 deletions(-) create mode 100644 airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json index 919cab1d149d..072dfd689231 100644 --- a/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { "customers": { - "updated_at": "1667643069899" + "updated_at": "11654622676000" } } diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json index 919cab1d149d..8535902fe849 100644 --- a/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { "customers": { - "updated_at": "1667643069899" + "updated_at": "1654622676000" } } diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py new file mode 100644 index 000000000000..0e6a1bf4d57a --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py @@ -0,0 +1,62 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from dataclasses import InitVar, dataclass +from typing import Any, ClassVar, Iterable, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.streams.core import Stream + + +@dataclass +class PartnerstackSlicer(StreamSlicer): + options: InitVar[Mapping[str, Any]] + cursor_field: str + request_cursor_field: str + + def __post_init__(self, options: Mapping[str, Any]): + self._state = {} + + def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, **kwargs) -> Iterable[StreamSlice]: + yield {self.request_cursor_field: stream_state.get(self.cursor_field, 0)} + + def _max_dt_str(self, *args: str) -> Optional[str]: + new_state_candidates = list(map(lambda x: int(x), filter(None, args))) + if not new_state_candidates: + return + max_dt = max(new_state_candidates) + return max_dt + + def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + slice_state = stream_slice.get(self.cursor_field) + current_state = self._state.get(self.cursor_field) + last_cursor = last_record and last_record[self.cursor_field] + max_dt = self._max_dt_str(slice_state, current_state, last_cursor) + if not max_dt: + return + self._state[self.cursor_field] = max_dt + + def get_stream_state(self) -> StreamState: + return self._state + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> MutableMapping[str, Any]: + return stream_slice or {} + + def get_request_headers(self, *args, **kwargs) -> Mapping[str, Any]: + return {} + + def get_request_body_data(self, *args, **kwargs) -> Optional[Union[Mapping, str]]: + return {} + + def get_request_body_json(self, *args, **kwargs) -> Optional[Mapping]: + return {} diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml index b4654ba73dc3..230485f29e46 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/partnerstack.yaml @@ -13,18 +13,11 @@ definitions: password: "{{ config['private_key'] }}" request_options_provider: request_parameters: - min_created: "{{ timestamp(stream_slice['start_time']) * 1000 }}" + min_created: "{{ timestamp(config['start_date']) * 1000 }}" stream_slicer: - type: "DatetimeStreamSlicer" - start_datetime: - datetime: "{{ config['start_date'] }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" - end_datetime: - datetime: "{{ now_utc() }}" - datetime_format: "%Y-%m-%d %H:%M:%S.%f%z" - step: "30d" - cursor_field: "{{ options.stream_cursor_field }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + request_cursor_field: "min_updated" + cursor_field: "updated_at" + class_name: source_partnerstack.components.PartnerstackSlicer retriever: record_selector: $ref: "*ref(definitions.selector)" From 11d28b8d1086747026cc5d0ab392349cdb8020a5 Mon Sep 17 00:00:00 2001 From: Elliot T Date: Mon, 7 Nov 2022 20:52:58 +0100 Subject: [PATCH 08/10] add incremental acceptances tests --- .../source-partnerstack/acceptance-test-config.yml | 11 ++++++++--- .../integration_tests/simple_catalog.json | 2 +- .../simple_incremental_catalog.json | 13 +++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_incremental_catalog.json diff --git a/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml index ed96b56e460d..adc1da7691b4 100644 --- a/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-config.yml @@ -17,9 +17,14 @@ acceptance_tests: basic_read: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [deals] + configured_catalog_path: "integration_tests/simple_catalog.json" + empty_streams: [] + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/simple_incremental_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/simple_catalog.json" diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json index e662b9017a9a..21d1624149f5 100644 --- a/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_catalog.json @@ -6,7 +6,7 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_incremental_catalog.json b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_incremental_catalog.json new file mode 100644 index 000000000000..e662b9017a9a --- /dev/null +++ b/airbyte-integrations/connectors/source-partnerstack/integration_tests/simple_incremental_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} From 36e53e79fba210646acee9ee9943b869fa2cbe09 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 7 Nov 2022 19:57:29 -0300 Subject: [PATCH 09/10] add partnerstack to source def --- .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ .../source-partnerstack/source_partnerstack/components.py | 4 +--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 57e5bd6856ff..ee1e5c7cb27f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -925,6 +925,13 @@ icon: pagerduty.svg sourceType: api releaseStage: alpha +- name: PartnerStack + sourceDefinitionId: d30fb809-6456-484d-8e2c-ee12e0f6888d + dockerRepository: airbyte/source-partnerstack + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/partnerstack + sourceType: api + releaseStage: alpha - name: Paypal Transaction sourceDefinitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 dockerRepository: airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py index 0e6a1bf4d57a..7b97596998b4 100644 --- a/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py +++ b/airbyte-integrations/connectors/source-partnerstack/source_partnerstack/components.py @@ -2,14 +2,12 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import datetime from dataclasses import InitVar, dataclass -from typing import Any, ClassVar, Iterable, Mapping, MutableMapping, Optional, Union +from typing import Any, Iterable, Mapping, MutableMapping, Optional, Union from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState -from airbyte_cdk.sources.streams.core import Stream @dataclass From fd796a743675271ab10d54cce7ffad0ea4400391 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 9 Nov 2022 00:17:29 +0000 Subject: [PATCH 10/10] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 82c04af80854..85752bc72aa3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -9244,6 +9244,39 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-partnerstack:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/partnerstack" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Partnerstack Spec" + type: "object" + required: + - "public_key" + - "private_key" + additionalProperties: true + properties: + public_key: + type: "string" + title: "Partnerstack Public key" + description: "The Live Public Key for a Partnerstack account." + airbyte_secret: true + private_key: + type: "string" + title: "Partnerstack Private key" + description: "The Live Private Key for a Partnerstack account." + airbyte_secret: true + start_date: + type: "string" + title: "Start date" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\ + \ data before this date will not be replicated." + examples: + - "2017-01-25T00:00:00Z" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-paypal-transaction:0.1.10" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/paypal-transactions"