From de70b410c8016b42a94506c6a535036f3d738fc8 Mon Sep 17 00:00:00 2001 From: Dainius Salkauskas Date: Fri, 28 Oct 2022 14:04:28 +0300 Subject: [PATCH 1/8] Init source smaily --- .../connectors/source-smaily/.dockerignore | 6 ++ .../connectors/source-smaily/Dockerfile | 38 +++++++++ .../connectors/source-smaily/README.md | 79 +++++++++++++++++++ .../connectors/source-smaily/__init__.py | 3 + .../source-smaily/acceptance-test-config.yml | 38 +++++++++ .../source-smaily/acceptance-test-docker.sh | 16 ++++ .../connectors/source-smaily/build.gradle | 9 +++ .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 16 ++++ .../integration_tests/configured_catalog.json | 13 +++ .../integration_tests/invalid_config.json | 5 ++ .../integration_tests/sample_config.json | 5 ++ .../integration_tests/sample_state.json | 5 ++ .../connectors/source-smaily/main.py | 13 +++ .../connectors/source-smaily/requirements.txt | 2 + .../connectors/source-smaily/setup.py | 29 +++++++ .../source-smaily/source_smaily/__init__.py | 8 ++ .../source_smaily/schemas/segments.json | 17 ++++ .../source_smaily/schemas/users.json | 34 ++++++++ .../source-smaily/source_smaily/smaily.yaml | 44 +++++++++++ .../source-smaily/source_smaily/source.py | 18 +++++ .../source-smaily/source_smaily/spec.yaml | 21 +++++ 22 files changed, 422 insertions(+) create mode 100644 airbyte-integrations/connectors/source-smaily/.dockerignore create mode 100644 airbyte-integrations/connectors/source-smaily/Dockerfile create mode 100644 airbyte-integrations/connectors/source-smaily/README.md create mode 100644 airbyte-integrations/connectors/source-smaily/__init__.py create mode 100644 airbyte-integrations/connectors/source-smaily/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-smaily/build.gradle create mode 100644 airbyte-integrations/connectors/source-smaily/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-smaily/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-smaily/main.py create mode 100644 airbyte-integrations/connectors/source-smaily/requirements.txt create mode 100644 airbyte-integrations/connectors/source-smaily/setup.py create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/__init__.py create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/source.py create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml diff --git a/airbyte-integrations/connectors/source-smaily/.dockerignore b/airbyte-integrations/connectors/source-smaily/.dockerignore new file mode 100644 index 000000000000..3e910911bfed --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_smaily +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-smaily/Dockerfile b/airbyte-integrations/connectors/source-smaily/Dockerfile new file mode 100644 index 000000000000..4eed4aa5d461 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_smaily ./source_smaily + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-smaily diff --git a/airbyte-integrations/connectors/source-smaily/README.md b/airbyte-integrations/connectors/source-smaily/README.md new file mode 100644 index 000000000000..3d33800341a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/README.md @@ -0,0 +1,79 @@ +# Smaily Source + +This is the repository for the Smaily configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/smaily). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-smaily:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/smaily) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_smaily/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source smaily test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-smaily:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-smaily:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-smaily:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smaily:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smaily:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-smaily:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-smaily:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-smaily:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-smaily/__init__.py b/airbyte-integrations/connectors/source-smaily/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-smaily/acceptance-test-config.yml b/airbyte-integrations/connectors/source-smaily/acceptance-test-config.yml new file mode 100644 index 000000000000..7e19a792a8c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/acceptance-test-config.yml @@ -0,0 +1,38 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-smaily:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_smaily/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-smaily/build.gradle b/airbyte-integrations/connectors/source-smaily/build.gradle new file mode 100644 index 000000000000..80cc96d71fa2 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_smaily' +} diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/__init__.py b/airbyte-integrations/connectors/source-smaily/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..e7961643f22b --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json new file mode 100644 index 000000000000..d99c1e7c471c --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_subdomain": "invalid_domain", + "api_username": "invalid_username", + "api_password": "invalid_password" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json new file mode 100644 index 000000000000..b0202e04e990 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_subdomain": "ea6l2xfv", + "api_username": "uvzmmg", + "api_password": "" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-smaily/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-smaily/main.py b/airbyte-integrations/connectors/source-smaily/main.py new file mode 100644 index 000000000000..d9d758860f7e --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smaily import SourceSmaily + +if __name__ == "__main__": + source = SourceSmaily() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-smaily/requirements.txt b/airbyte-integrations/connectors/source-smaily/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-smaily/setup.py b/airbyte-integrations/connectors/source-smaily/setup.py new file mode 100644 index 000000000000..5193a5b2eb6d --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_smaily", + description="Source implementation for Smaily.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/__init__.py b/airbyte-integrations/connectors/source-smaily/source_smaily/__init__.py new file mode 100644 index 000000000000..4a74fb5c7ad2 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceSmaily + +__all__ = ["SourceSmaily"] diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json new file mode 100644 index 000000000000..f045c35a2908 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json @@ -0,0 +1,17 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": ["string", "null"] + }, + "subscribers_count": { + "type": ["integer", "null"] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json new file mode 100644 index 000000000000..933abcec70d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json @@ -0,0 +1,34 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "email": { + "type": ["string", "null"] + }, + "first_name": { + "type": ["string", "null"] + }, + "last_name": { + "type": ["string", "null"] + }, + "is_owner": { + "type": ["boolean", "null"] + }, + "role": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": ["string", "null"] + } + } + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml b/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml new file mode 100644 index 000000000000..59395f2da670 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml @@ -0,0 +1,44 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://{{config['api_subdomain']}}.sendsmaily.net/api" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{config['api_username']}}" + password: "{{config['api_password']}}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + # API Docs: https://smaily.com/help/api/organizations/list-users-of-an-organization/ + users_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "users" + primary_key: "id" + path: "/organizations/users.php" + # API Docs: https://smaily.com/help/api/segments/list-segments/ + segments_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "segments" + primary_key: "id" + path: "/list.php" + +streams: + - "*ref(definitions.users_stream)" + - "*ref(definitions.segments_stream)" + +check: + stream_names: ["users"] diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/source.py b/airbyte-integrations/connectors/source-smaily/source_smaily/source.py new file mode 100644 index 000000000000..9104fe3575ee --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceSmaily(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "smaily.yaml"}) diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml b/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml new file mode 100644 index 000000000000..1a477b527565 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml @@ -0,0 +1,21 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/smaily +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Smaily Spec + type: object + required: + - api_subdomain + - api_username + - api_password + additionalProperties: true + properties: + api_subdomain: + type: string + description: API Subdomain + api_username: + type: string + description: API user username + api_password: + type: string + description: API user password + airbyte_secret: true \ No newline at end of file From b7a5c5276fda8362de9ef06b6ed94adbea9a8f5b Mon Sep 17 00:00:00 2001 From: Dainius Salkauskas Date: Fri, 28 Oct 2022 19:04:59 +0300 Subject: [PATCH 2/8] Added campaigns, templates, automations, ab_tests streams --- .../integration_tests/configured_catalog.json | 61 ++++++++++++++++- .../source_smaily/schemas/ab_tests.json | 60 +++++++++++++++++ .../source_smaily/schemas/automations.json | 60 +++++++++++++++++ .../source_smaily/schemas/campaigns.json | 67 +++++++++++++++++++ .../source_smaily/schemas/templates.json | 26 +++++++ .../source-smaily/source_smaily/smaily.yaml | 32 +++++++++ 6 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json create mode 100644 airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json index e7961643f22b..8f1b6d743615 100644 --- a/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json @@ -4,10 +4,67 @@ "stream": { "name": "users", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "segments", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "templates", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "automations", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "ab_tests", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh" + ] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json new file mode 100644 index 000000000000..397d476647cf --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json @@ -0,0 +1,60 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": ["string", "null"] + }, + "sections": { + "type": ["array", "null"], + "items": + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "subject": { + "type": ["string", "null"] + }, + "template": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": ["string", "null"] + }, + "preview_url": { + "type": ["string", "null"] + } + } + } + } + } + }, + "tags": { + "type": ["array", "null"], + "items": + { + "type": ["string", "null"] + } + + }, + "created_at": { + "type": ["string", "null"] + }, + "completed_at": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json new file mode 100644 index 000000000000..8b97d6b91354 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json @@ -0,0 +1,60 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "sections": { + "type": [ + "array", + "null" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "subject": { + "type": [ + "string", + "null" + ] + }, + "template": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "preview_url": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json new file mode 100644 index 000000000000..0a54784ceb80 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json @@ -0,0 +1,67 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "template": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "preview_url": { + "type": [ + "string", + "null" + ] + } + } + }, + "tags": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "created_at": { + "type": [ + "string", + "null" + ] + }, + "completed_at": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json new file mode 100644 index 000000000000..4c37f0910c29 --- /dev/null +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json @@ -0,0 +1,26 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "type": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["string", "null"] + }, + "modified_at": { + "type": ["string", "null"] + }, + "preview_url": { + "type": ["string", "null"] + } + }, + "required": [ + "id" + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml b/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml index 59395f2da670..98adf9a932a2 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml @@ -35,10 +35,42 @@ definitions: name: "segments" primary_key: "id" path: "/list.php" + # API Docs: https://smaily.com/help/api/campaigns-3/list-campaigns/ + campaigns_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "campaigns" + primary_key: "id" + path: "/campaign.php" + # API Docs: https://smaily.com/help/api/templates-2/list-templates/ + templates_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "templates" + primary_key: "id" + path: "/templates.php" + # API Docs: https://smaily.com/help/api/automations-2/list-automation-workflows/ + automations_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "automations" + primary_key: "id" + path: "/autoresponder.php" + # API Docs: https://smaily.com/help/api/a-b-tests/list-a-b-tests/ + ab_tests_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "ab_tests" + primary_key: "id" + path: "/split.php" streams: - "*ref(definitions.users_stream)" - "*ref(definitions.segments_stream)" + - "*ref(definitions.campaigns_stream)" + - "*ref(definitions.templates_stream)" + - "*ref(definitions.automations_stream)" + - "*ref(definitions.ab_tests_stream)" check: stream_names: ["users"] From dc0bcc7386a5b2f8f36821bcffeca1e420c8fcad Mon Sep 17 00:00:00 2001 From: Dainius Salkauskas Date: Sun, 30 Oct 2022 22:40:37 +0200 Subject: [PATCH 3/8] Added pagination --- .../source-smaily/source_smaily/smaily.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml b/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml index 98adf9a932a2..e52dec4b7dc0 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/smaily.yaml @@ -11,6 +11,18 @@ definitions: type: BasicHttpAuthenticator username: "{{config['api_username']}}" password: "{{config['api_password']}}" + increment_paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: PageIncrement + page_size: 250 + page_token_option: + inject_into: "request_parameter" + field_name: "page" retriever: record_selector: $ref: "*ref(definitions.selector)" @@ -28,6 +40,10 @@ definitions: name: "users" primary_key: "id" path: "/organizations/users.php" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" # API Docs: https://smaily.com/help/api/segments/list-segments/ segments_stream: $ref: "*ref(definitions.base_stream)" @@ -49,6 +65,13 @@ definitions: name: "templates" primary_key: "id" path: "/templates.php" + retriever: + $ref: "*ref(definitions.retriever)" + paginator: + $ref: "*ref(definitions.increment_paginator)" + pagination_strategy: + type: PageIncrement + page_size: 1000 # API Docs: https://smaily.com/help/api/automations-2/list-automation-workflows/ automations_stream: $ref: "*ref(definitions.base_stream)" From d5e5d7f35e35f7c5e4812557b7fc00215026ca49 Mon Sep 17 00:00:00 2001 From: Dainius Salkauskas Date: Sun, 30 Oct 2022 22:49:37 +0200 Subject: [PATCH 4/8] Added documentation --- .../source-smaily/source_smaily/spec.yaml | 6 +-- docs/integrations/sources/smaily.md | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 docs/integrations/sources/smaily.md diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml b/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml index 1a477b527565..8320c6e37bfe 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml @@ -11,11 +11,11 @@ connectionSpecification: properties: api_subdomain: type: string - description: API Subdomain + description: API Subdomain. See https://smaily.com/help/api/general/create-api-user/ api_username: type: string - description: API user username + description: API user username. See https://smaily.com/help/api/general/create-api-user/ api_password: type: string - description: API user password + description: API user password. See https://smaily.com/help/api/general/create-api-user/ airbyte_secret: true \ No newline at end of file diff --git a/docs/integrations/sources/smaily.md b/docs/integrations/sources/smaily.md new file mode 100644 index 000000000000..667f6793d019 --- /dev/null +++ b/docs/integrations/sources/smaily.md @@ -0,0 +1,39 @@ +# Smaily + +## Sync overview + +This source can sync data from the [Smaily API](https://smaily.com/help/api/). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. + +## This Source Supports the Following Streams + +* users +* segments +* campaigns +* templates +* automations +* A/B tests + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | + +### Performance considerations + +The connector has a rate limit of 5 API requests per second per IP-address. + +## Getting started + +### Requirements + +* Smaily API user username +* Smaily API user password +* Smaily API subdomain + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :----------- |:-----------------------------------------------------------| +| 0.1.0 | 2022-10-25 | [TODO](TODO) | Initial commit | \ No newline at end of file From 4fb5a2e8085b119a84bd9fedd37f4e17e10997d7 Mon Sep 17 00:00:00 2001 From: Dainius Salkauskas Date: Sun, 30 Oct 2022 23:03:43 +0200 Subject: [PATCH 5/8] Updated PR information --- docs/integrations/sources/smaily.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/smaily.md b/docs/integrations/sources/smaily.md index 667f6793d019..19d817011093 100644 --- a/docs/integrations/sources/smaily.md +++ b/docs/integrations/sources/smaily.md @@ -36,4 +36,4 @@ The connector has a rate limit of 5 API requests per second per IP-address. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------- |:-----------------------------------------------------------| -| 0.1.0 | 2022-10-25 | [TODO](TODO) | Initial commit | \ No newline at end of file +| 0.1.0 | 2022-10-25 | [18674](https://github.com/airbytehq/airbyte/pull/18674) | Initial commit | \ No newline at end of file From b606ea9f3e42350631636a02e7c2deef1ec33125 Mon Sep 17 00:00:00 2001 From: Dainius Salkauskas Date: Mon, 31 Oct 2022 15:08:09 +0200 Subject: [PATCH 6/8] Added titles in spec.yaml, formatted json files --- .../source-smaily/integration_tests/configured_catalog.json | 2 +- .../source-smaily/integration_tests/invalid_config.json | 2 +- .../source-smaily/integration_tests/sample_config.json | 2 +- .../source-smaily/source_smaily/schemas/ab_tests.json | 2 +- .../source-smaily/source_smaily/schemas/automations.json | 2 +- .../source-smaily/source_smaily/schemas/campaigns.json | 2 +- .../source-smaily/source_smaily/schemas/segments.json | 2 +- .../source-smaily/source_smaily/schemas/templates.json | 2 +- .../source-smaily/source_smaily/schemas/users.json | 2 +- .../connectors/source-smaily/source_smaily/spec.yaml | 5 ++++- 10 files changed, 13 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json index 8f1b6d743615..500cb98db4f7 100644 --- a/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/configured_catalog.json @@ -67,4 +67,4 @@ "destination_sync_mode": "overwrite" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json index d99c1e7c471c..29439fa04279 100644 --- a/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/invalid_config.json @@ -2,4 +2,4 @@ "api_subdomain": "invalid_domain", "api_username": "invalid_username", "api_password": "invalid_password" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json index b0202e04e990..6239900e68d6 100644 --- a/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-smaily/integration_tests/sample_config.json @@ -2,4 +2,4 @@ "api_subdomain": "ea6l2xfv", "api_username": "uvzmmg", "api_password": "" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json index 397d476647cf..2e77511f34e5 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/ab_tests.json @@ -57,4 +57,4 @@ "required": [ "id" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json index 8b97d6b91354..a2cf811f25d3 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/automations.json @@ -57,4 +57,4 @@ "required": [ "id" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json index 0a54784ceb80..27e7e6fe79d5 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/campaigns.json @@ -64,4 +64,4 @@ "required": [ "id" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json index f045c35a2908..2d2931c6ad4d 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/segments.json @@ -14,4 +14,4 @@ "required": [ "id" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json index 4c37f0910c29..3e608cea3386 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/templates.json @@ -23,4 +23,4 @@ "required": [ "id" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json index 933abcec70d2..6e50e08d8475 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/schemas/users.json @@ -31,4 +31,4 @@ "required": [ "id" ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml b/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml index 8320c6e37bfe..d22c139eb709 100644 --- a/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml +++ b/airbyte-integrations/connectors/source-smaily/source_smaily/spec.yaml @@ -11,11 +11,14 @@ connectionSpecification: properties: api_subdomain: type: string + title: API Subdomain description: API Subdomain. See https://smaily.com/help/api/general/create-api-user/ api_username: type: string + title: API User Username description: API user username. See https://smaily.com/help/api/general/create-api-user/ api_password: type: string + title: API User Password description: API user password. See https://smaily.com/help/api/general/create-api-user/ - airbyte_secret: true \ No newline at end of file + airbyte_secret: true From cb1a1def0accb6f0221f42e79ed11a55695df532 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 8 Nov 2022 21:58:20 -0300 Subject: [PATCH 7/8] add smaily to source def --- .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index ee98e6bcba5d..ef93086f137b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1217,6 +1217,13 @@ icon: slack.svg sourceType: api releaseStage: generally_available +- name: Smaily + sourceDefinitionId: 781f8b1d-4e20-4842-a2c3-cd9b119d65fa + dockerRepository: airbyte/source-smaily + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/smaily + sourceType: api + releaseStage: alpha - name: Smartsheets sourceDefinitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc dockerRepository: airbyte/source-smartsheets From d634a8c9d28b23822d2722d675c8da0451fc0815 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Wed, 9 Nov 2022 01:33:00 +0000 Subject: [PATCH 8/8] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index b16333418db1..71f08bbcbf72 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11695,6 +11695,35 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" +- dockerImage: "airbyte/source-smaily:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/smaily" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Smaily Spec" + type: "object" + required: + - "api_subdomain" + - "api_username" + - "api_password" + additionalProperties: true + properties: + api_subdomain: + type: "string" + title: "API Subdomain" + description: "API Subdomain. See https://smaily.com/help/api/general/create-api-user/" + api_username: + type: "string" + title: "API User Username" + description: "API user username. See https://smaily.com/help/api/general/create-api-user/" + api_password: + type: "string" + title: "API User Password" + description: "API user password. See https://smaily.com/help/api/general/create-api-user/" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-smartsheets:0.1.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/smartsheets"