From f6bd354f0f1c4d1353cbfcd68db988dd46f6c8a9 Mon Sep 17 00:00:00 2001 From: akashpatl Date: Wed, 9 Nov 2022 21:35:45 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20Younium=20[pytho?= =?UTF-8?q?n=20cdk]=20=20(#18758)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add younium source along with documentation * add documentationUrl * remove newlines eof * remove additional newlines * remove TODO file * add titles to all params in spec.yaml * add younium to source def * run format * auto-bump connector version Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 34 + .../connectors/source-younium/.dockerignore | 6 + .../connectors/source-younium/Dockerfile | 38 + .../connectors/source-younium/README.md | 132 +++ .../source-younium/acceptance-test-config.yml | 26 + .../source-younium/acceptance-test-docker.sh | 15 + .../connectors/source-younium/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 + .../integration_tests/configured_catalog.json | 31 + .../integration_tests/invalid_config.json | 6 + .../integration_tests/sample_config.json | 6 + .../integration_tests/sample_state.json | 5 + .../connectors/source-younium/main.py | 13 + .../source-younium/requirements.txt | 2 + .../connectors/source-younium/setup.py | 30 + .../source-younium/source_younium/__init__.py | 8 + .../source_younium/schemas/invoice.json | 386 ++++++++ .../source_younium/schemas/product.json | 174 ++++ .../source_younium/schemas/subscription.json | 836 ++++++++++++++++++ .../source-younium/source_younium/source.py | 131 +++ .../source-younium/source_younium/spec.yaml | 28 + .../source-younium/unit_tests/__init__.py | 3 + .../source-younium/unit_tests/test_source.py | 51 ++ .../source-younium/unit_tests/test_streams.py | 73 ++ docs/integrations/sources/younium.md | 36 + 28 files changed, 2110 insertions(+) create mode 100644 airbyte-integrations/connectors/source-younium/.dockerignore create mode 100644 airbyte-integrations/connectors/source-younium/Dockerfile create mode 100644 airbyte-integrations/connectors/source-younium/README.md create mode 100644 airbyte-integrations/connectors/source-younium/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-younium/build.gradle create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-younium/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-younium/main.py create mode 100644 airbyte-integrations/connectors/source-younium/requirements.txt create mode 100644 airbyte-integrations/connectors/source-younium/setup.py create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/__init__.py create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/source.py create mode 100644 airbyte-integrations/connectors/source-younium/source_younium/spec.yaml create mode 100644 airbyte-integrations/connectors/source-younium/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-younium/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/younium.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9f4455497d51..25f0ff7bcba1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1633,6 +1633,13 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/yandex-metrica sourceType: api releaseStage: alpha +- name: Younium + sourceDefinitionId: 9c74c2d7-531a-4ebf-b6d8-6181f805ecdc + dockerRepository: airbyte/source-younium + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/younium + sourceType: api + releaseStage: alpha - name: Zoom sourceDefinitionId: cbfd9856-1322-44fb-bcf1-0b39b7a8e92e dockerRepository: airbyte/source-zoom diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 6c145efa9700..69d9e963def3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -15153,6 +15153,40 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-younium:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/younium" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Younium Spec" + type: "object" + required: + - "username" + - "password" + - "legal_entity" + properties: + username: + title: "Username" + type: "string" + description: "Username for Younium account" + password: + title: "Password" + type: "string" + description: "Account password for younium account API key" + airbyte_secret: true + legal_entity: + title: "Legal Entity" + type: "string" + description: "Legal Entity that data should be pulled from" + playground: + title: "Playground environment" + type: "boolean" + description: "Property defining if connector is used against playground\ + \ or production environment" + default: false + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-zoom:0.1.0" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/zoom" diff --git a/airbyte-integrations/connectors/source-younium/.dockerignore b/airbyte-integrations/connectors/source-younium/.dockerignore new file mode 100644 index 000000000000..cebe350aeec3 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_younium +!setup.py +!secrets \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/Dockerfile b/airbyte-integrations/connectors/source-younium/Dockerfile new file mode 100644 index 000000000000..92fb46be3954 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_younium ./source_younium + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-younium \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/README.md b/airbyte-integrations/connectors/source-younium/README.md new file mode 100644 index 000000000000..0c8f7fa797d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/README.md @@ -0,0 +1,132 @@ +# Younium Source + +This is the repository for the Younium source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/younium). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-younium:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/younium) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_younium/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source younium test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-younium:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-younium:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-younium:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-younium:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-younium:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-younium:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-younium:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-younium:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml b/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml new file mode 100644 index 000000000000..56827d2bf7c3 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-younium:dev +tests: + spec: + - spec_path: "source_younium/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh new file mode 100755 index 000000000000..fa680528f222 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/build.gradle b/airbyte-integrations/connectors/source-younium/build.gradle new file mode 100644 index 000000000000..81baf943b54e --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_younium' +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/__init__.py b/airbyte-integrations/connectors/source-younium/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-younium/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..a9a071c44c44 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "invoice", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "product", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "subscription", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-younium/integration_tests/invalid_config.json new file mode 100644 index 000000000000..7a61623e3ac9 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "username": "invalid username", + "password": "invalid password", + "legal_entity": "invalid legal entity", + "playground": "invalid flag" +} diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-younium/integration_tests/sample_config.json new file mode 100644 index 000000000000..2935f0355270 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "username": "user@user.com", + "password": "password", + "legal_entity": "XX co", + "playground": true +} diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-younium/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-younium/main.py b/airbyte-integrations/connectors/source-younium/main.py new file mode 100644 index 000000000000..d638d3d94c3f --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_younium import SourceYounium + +if __name__ == "__main__": + source = SourceYounium() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-younium/requirements.txt b/airbyte-integrations/connectors/source-younium/requirements.txt new file mode 100644 index 000000000000..78140e52009f --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/setup.py b/airbyte-integrations/connectors/source-younium/setup.py new file mode 100644 index 000000000000..9c1c848e788c --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "responses~=0.22.0", + "source-acceptance-test", +] + +setup( + name="source_younium", + description="Source implementation for Younium.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-younium/source_younium/__init__.py b/airbyte-integrations/connectors/source-younium/source_younium/__init__.py new file mode 100644 index 000000000000..e51d99b37ef6 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceYounium + +__all__ = ["SourceYounium"] diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json new file mode 100644 index 000000000000..96c65fe57d0c --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json @@ -0,0 +1,386 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "invoiceNumber": { + "type": "string" + }, + "status": { + "type": "string" + }, + "account": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "accountNumber": { + "type": "string" + }, + "id": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "notes": { + "type": "string" + }, + "invoiceDate": { + "type": "string" + }, + "dueDate": { + "type": "string" + }, + "daysPastDue": { + "type": "number" + }, + "nrOfReminders": { + "type": "number" + }, + "paymentTerm": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "days": { + "type": "number" + }, + "name": { + "type": "string" + } + } + }, + "currency": { + "type": "string" + }, + "subtotal": { + "type": "number" + }, + "tax": { + "type": "number" + }, + "totalAmount": { + "type": "number" + }, + "totalRoundingAmount": { + "type": "number" + }, + "settledAmount": { + "type": "number" + }, + "balancedAmount": { + "type": "number" + }, + "taxIncluded": { + "type": "boolean" + }, + "invoiceAddress": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "name": { + "type": "string" + }, + "street": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "county": { + "type": "string" + }, + "state": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "country": { + "type": "string" + } + } + }, + "deliveryAddress": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "name": { + "type": "string" + }, + "street": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "county": { + "type": "string" + }, + "state": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "country": { + "type": "string" + } + } + }, + "invoiceBatchId": { + "type": "string" + }, + "invoiceLines": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "invoiceLineNumber": { + "type": "number" + }, + "productNumber": { + "type": "string" + }, + "productName": { + "type": "string" + }, + "chargeDescription": { + "type": "string" + }, + "chargeNumber": { + "type": "string" + }, + "quantity": { + "type": "number" + }, + "unitOfMeasure": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "unitCode": { + "type": "string" + }, + "name": { + "type": "string" + }, + "displayName": { + "type": "string" + } + } + }, + "price": { + "type": "number" + }, + "subtotal": { + "type": "number" + }, + "total": { + "type": "number" + }, + "tax": { + "type": "number" + }, + "servicePeriodStartDate": { + "type": "string" + }, + "servicePeriodEndDate": { + "type": "string" + }, + "notes": { + "type": "string" + }, + "orderChargeId": { + "type": "string" + }, + "orderId": { + "type": "string" + }, + "accountId": { + "type": "string" + }, + "customFields": { + "type": "object" + }, + "accountsReceivable": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "code": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "deferredRevenue": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "code": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "recognizedRevenue": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "code": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + }, + "taxCategoryName": { + "type": "string" + }, + "taxRate": { + "type": "number" + } + } + } + }, + "yourReference": { + "type": "string" + }, + "ourReference": { + "type": "string" + }, + "yourOrderNumber": { + "type": "string" + }, + "buyerReference": { + "type": "string" + }, + "invoiceType": { + "type": "string" + }, + "sendMethod": { + "type": "string" + }, + "exchangeRate": { + "type": "number" + }, + "settledNotes": { + "type": "string" + }, + "invoiceTemplateId": { + "type": "string" + }, + "disableAutomaticInvoiceReminder": { + "type": "boolean" + }, + "onlinePaymentLink": { + "type": "string" + }, + "accountsReceivable": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "code": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "customFields": { + "type": "object" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json new file mode 100644 index 000000000000..296946ccd16d --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json @@ -0,0 +1,174 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "productNumber": { + "type": "string" + }, + "name": { + "type": "string" + }, + "productType": { + "type": "string" + }, + "category": { + "type": "string" + }, + "activationDate": { + "type": "string" + }, + "endOfNewSalesDate": { + "type": "string" + }, + "endOfRenewalDate": { + "type": "string" + }, + "endOfLifeDate": { + "type": "string" + }, + "isFrameworkProduct": { + "type": "boolean" + }, + "chargePlans": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chargePlanNumber": { + "type": "string" + }, + "name": { + "type": "string" + }, + "effectiveStartDate": { + "type": "string" + }, + "endOfNewSalesDate": { + "type": "string" + }, + "effectiveEndDate": { + "type": "string" + }, + "charges": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chargeNumber": { + "type": "string" + }, + "name": { + "type": "string" + }, + "model": { + "type": "string" + }, + "chargeType": { + "type": "string" + }, + "unitCode": { + "type": "string" + }, + "defaultQuantity": { + "type": "number" + }, + "pricePeriod": { + "type": "string" + }, + "usageRating": { + "type": "string" + }, + "createInvoiceLinesPerTier": { + "type": "boolean" + }, + "billingDay": { + "type": "string" + }, + "specificBillingDay": { + "type": "number" + }, + "billingPeriod": { + "type": "string" + }, + "periodAlignment": { + "type": "string" + }, + "billingTiming": { + "type": "string" + }, + "taxTemplate": { + "type": "string" + }, + "taxIncluded": { + "type": "boolean" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + }, + "deferredRevenueAccount": { + "type": "string" + }, + "recognizedRevenueAccount": { + "type": "string" + }, + "customFields": { + "type": "object" + }, + "priceDetails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "currency": { + "type": "string" + }, + "price": { + "type": "number" + }, + "tier": { + "type": "number" + }, + "description": { + "type": "string" + }, + "fromQuantity": { + "type": "number" + }, + "toQuantity": { + "type": "number" + }, + "priceBase": { + "type": "string" + } + } + } + } + } + } + } + } + } + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + }, + "customFields": { + "type": "object" + } + } +} diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json new file mode 100644 index 000000000000..c4ea6eeca3ff --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json @@ -0,0 +1,836 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "orderNumber": { + "type": "string" + }, + "version": { + "type": "number" + }, + "isLastVersion": { + "type": "boolean" + }, + "status": { + "type": "string" + }, + "description": { + "type": "string" + }, + "remarks": { + "type": "string" + }, + "effectiveStartDate": { + "type": "string" + }, + "effectiveEndDate": { + "type": "string" + }, + "cancellationDate": { + "type": "string" + }, + "effectiveChangeDate": { + "type": "string" + }, + "orderDate": { + "type": "string" + }, + "noticePeriodDate": { + "type": "string" + }, + "lastRenewalDate": { + "type": "string" + }, + "noticePeriod": { + "type": "number" + }, + "term": { + "type": "number" + }, + "renewalTerm": { + "type": "number" + }, + "isAutoRenewed": { + "type": "boolean" + }, + "orderType": { + "type": "string" + }, + "termType": { + "type": "string" + }, + "orderPaymentMethod": { + "type": "string" + }, + "invoiceSeparatly": { + "type": "boolean" + }, + "yourReference": { + "type": "string" + }, + "ourReference": { + "type": "string" + }, + "yourOrderNumber": { + "type": "string" + }, + "invoiceAddress": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "name": { + "type": "string" + }, + "street": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "county": { + "type": "string" + }, + "state": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "country": { + "type": "string" + } + } + }, + "deliveryAddress": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "description": { + "type": "string" + }, + "name": { + "type": "string" + }, + "street": { + "type": "string" + }, + "street2": { + "type": "string" + }, + "city": { + "type": "string" + }, + "county": { + "type": "string" + }, + "state": { + "type": "string" + }, + "zip": { + "type": "string" + }, + "country": { + "type": "string" + } + } + }, + "invoiceBatchGroup": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "code": { + "type": "string" + }, + "description": { + "type": "string" + } + } + }, + "paymentTerm": { + "type": "string" + }, + "useAccountInvoiceBatchGroup": { + "type": "boolean" + }, + "account": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "accountNumber": { + "type": "string" + }, + "id": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "invoiceAccount": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "accountNumber": { + "type": "string" + }, + "id": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + } + } + }, + "products": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "productNumber": { + "type": "string" + }, + "chargePlanId": { + "type": "string" + }, + "chargePlanName": { + "type": "string" + }, + "chargePlanNumber": { + "type": "string" + }, + "productLineNumber": { + "type": "number" + }, + "name": { + "type": "string" + }, + "charges": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chargeNumber": { + "type": "string" + }, + "version": { + "type": "number" + }, + "isLastVersion": { + "type": "boolean" + }, + "name": { + "type": "string" + }, + "chargeType": { + "type": "string" + }, + "priceModel": { + "type": "string" + }, + "effectiveStartDate": { + "type": "string" + }, + "effectiveEndDate": { + "type": "string" + }, + "quantity": { + "type": "number" + }, + "unitCode": { + "type": "string" + }, + "startOn": { + "type": "string" + }, + "endOn": { + "type": "string" + }, + "chargedThroughDate": { + "type": "string" + }, + "lastRenewalDate": { + "type": "string" + }, + "lastPriceAdjustmentDate": { + "type": "string" + }, + "pricePeriod": { + "type": "string" + }, + "usageRating": { + "type": "string" + }, + "revenueRecognitionRule": { + "type": "string" + }, + "billingDay": { + "type": "string" + }, + "specificBillingDay": { + "type": "number" + }, + "billingPeriod": { + "type": "string" + }, + "billingTiming": { + "type": "string" + }, + "periodAlignment": { + "type": "string" + }, + "taxTemplate": { + "type": "string" + }, + "taxIncluded": { + "type": "boolean" + }, + "createInvoiceLinesPerTier": { + "type": "boolean" + }, + "estimatedUsage": { + "type": "number" + }, + "estimatedQuantity": { + "type": "number" + }, + "remarks": { + "type": "string" + }, + "accountsReceivableAccount": { + "type": "string" + }, + "deferredRevenueAccount": { + "type": "string" + }, + "recognizedRevenueAccount": { + "type": "string" + }, + "changeState": { + "type": "string" + }, + "displayPrice": { + "type": "number" + }, + "customFields": { + "type": "object" + }, + "priceDetails": { + "type": "array", + "items": { + "type": "object", + "properties": { + "tier": { + "type": "number" + }, + "price": { + "type": "number" + }, + "listPrice": { + "type": "number" + }, + "description": { + "type": "string" + }, + "fromQuantity": { + "type": "number" + }, + "toQuantity": { + "type": "number" + }, + "priceBase": { + "type": "string" + }, + "lineDiscountPercent": { + "type": "number" + }, + "lineDiscountAmount": { + "type": "number" + } + } + } + }, + "recurringMonthlyAmount": { + "type": "number" + }, + "recurringMonthlyAmountBase": { + "type": "number" + }, + "features": { + "type": "array", + "items": { + "type": "object", + "properties": { + "code": { + "type": "string" + }, + "description": { + "type": "string" + } + } + } + }, + "orderDiscounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "orderDiscountId": { + "type": "string" + }, + "chargeId": { + "type": "string" + } + } + } + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + }, + "cmrr": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "acv": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "tcv": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "emrr": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "oneTimeFees": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "orderProductId": { + "type": "string" + }, + "orderId": { + "type": "string" + } + } + } + }, + "customFields": { + "type": "object" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + }, + "cmrr": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "acv": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "emrr": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "oneTimeFees": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "tcv": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + } + } + } + }, + "milestones": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "orderId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "milestoneDate": { + "type": "string" + }, + "plannedDate": { + "type": "string" + } + } + } + }, + "orderDiscounts": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "orderId": { + "type": "string" + }, + "startOn": { + "type": "string" + }, + "endOn": { + "type": "string" + }, + "startDate": { + "type": "string" + }, + "endDate": { + "type": "string" + }, + "percent": { + "type": "number" + }, + "discountType": { + "type": "string" + }, + "orderProductCharges": { + "type": "array", + "items": { + "type": "object", + "properties": { + "orderDiscountId": { + "type": "string" + }, + "chargeId": { + "type": "string" + } + } + } + }, + "onSpecificCharges": { + "type": "boolean" + } + } + } + }, + "currency": { + "type": "string" + }, + "externalERPId": { + "type": "string" + }, + "externalCRMId": { + "type": "string" + }, + "currencyCodeToUseWhenInvoice": { + "type": "string" + }, + "customFields": { + "type": "object" + }, + "cmrr": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "acv": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "emrr": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "oneTimeFees": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + }, + "tcv": { + "type": "object", + "properties": { + "amount": { + "type": "number" + }, + "currencyCode": { + "type": "string" + }, + "currencyConversionDate": { + "type": "string" + }, + "baseCurrencyAmount": { + "type": "number" + }, + "baseCurrencyCode": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-younium/source_younium/source.py b/airbyte-integrations/connectors/source-younium/source_younium/source.py new file mode 100644 index 000000000000..ac4c5be6a632 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/source.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +# Basic full refresh stream +class YouniumStream(HttpStream, ABC): + # url_base = "https://apisandbox.younium.com" + + # https://api.younium.com + def __init__(self, authenticator=TokenAuthenticator, playground: bool = False, *args, **kwargs): + super().__init__(authenticator=authenticator) + self.page_size = 100 + self.playground: bool = playground + + @property + def url_base(self) -> str: + if self.playground: + endpoint = "https://apisandbox.younium.com" + else: + endpoint = "https://api.younium.com" + return endpoint + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + response = response.json() + current_page = response.get("pageNumber", 1) + total_rows = response.get("totalCount", 0) + + total_pages = total_rows // self.page_size + + if current_page <= total_pages: + return {"pageNumber": current_page + 1} + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + if next_page_token: + return {"pageNumber": next_page_token["pageNumber"], "PageSize": self.page_size} + else: + return {"PageSize": self.page_size} + + def parse_response( + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + response_results = response.json() + yield from response_results.get("data", []) + + +class Invoice(YouniumStream): + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "Invoices" + + +class Product(YouniumStream): + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "Products" + + +class Subscription(YouniumStream): + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "Subscriptions" + + +class SourceYounium(AbstractSource): + def get_auth(self, config): + scope = "openid youniumapi profile" + + if config.get("playground"): + url = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" + else: + url = "https://younium-identity-server.azurewebsites.net/connect/token" + + payload = f"grant_type=password&client_id=apiclient&username={config['username']}&password={config['password']}&scope={scope}" + headers = {"Content-Type": "application/x-www-form-urlencoded"} + response = requests.request("POST", url, headers=headers, data=payload) + response.raise_for_status() + access_token = response.json()["access_token"] + + auth = TokenAuthenticator(token=access_token) + return auth + + def check_connection(self, logger, config) -> Tuple[bool, any]: + + try: + stream = Invoice(authenticator=self.get_auth(config), **config) + stream.next_page_token = lambda response: None + stream.page_size = 1 + # auth = self.get_auth(config) + _ = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + return True, None + except Exception as e: + logger.error(e) + return False, repr(e) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + auth = self.get_auth(config) + return [Invoice(authenticator=auth, **config), Product(authenticator=auth, **config), Subscription(authenticator=auth, **config)] diff --git a/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml b/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml new file mode 100644 index 000000000000..0cf38ff69a03 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml @@ -0,0 +1,28 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/younium +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Younium Spec + type: object + required: + - username + - password + - legal_entity + properties: + username: + title: Username + type: string + description: Username for Younium account + password: + title: Password + type: string + description: Account password for younium account API key + airbyte_secret: true + legal_entity: + title: Legal Entity + type: string + description: Legal Entity that data should be pulled from + playground: + title: Playground environment + type: boolean + description: Property defining if connector is used against playground or production environment + default: false diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/__init__.py b/airbyte-integrations/connectors/source-younium/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py b/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py new file mode 100644 index 000000000000..61f8dc242c89 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py @@ -0,0 +1,51 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import responses +from source_younium.source import SourceYounium + + +@responses.activate +def test_check_connection(mocker): + sandbox = False + + source = SourceYounium() + # mock the post request + + if sandbox: + mock_url1 = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" + mock_url2 = "https://apisandbox.younium.com/Invoices?PageSize=1" + else: + mock_url1 = "https://younium-identity-server.azurewebsites.net/connect/token" + mock_url2 = "https://api.younium.com/Invoices?PageSize=1" + # Mock the POST to get the access token + responses.add( + responses.POST, + mock_url1, + json={ + "access_token": "dummy_token", + }, + status=HTTPStatus.OK, + ) + + # Mock the GET to get the first page of the stream + responses.add(responses.GET, mock_url2, json={}, status=HTTPStatus.OK) + + logger_mock = MagicMock() + config_mock = {"playground": sandbox, "username": "dummy_username", "password": "dummy_password"} + + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceYounium() + mocker.patch.object(source, "get_auth", return_value="dummy_token") + config_mock = {"playground": False, "username": "dummy_username", "password": "dummy_password"} + streams = source.streams(config_mock) + + expected_streams_number = 3 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py new file mode 100644 index 000000000000..39d4621a927b --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_younium.source import YouniumStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(YouniumStream, "path", "v0/example_endpoint") + mocker.patch.object(YouniumStream, "primary_key", "test_primary_key") + mocker.patch.object(YouniumStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = YouniumStream(authenticator=None) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {"PageSize": 100} + assert stream.request_params(**inputs) == expected_params + + +def test_request_params_with_next_page_token(patch_base_class): + stream = YouniumStream(authenticator=None) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"pageNumber": 2}} + expected_params = {"PageSize": 100, "pageNumber": 2} + assert stream.request_params(**inputs) == expected_params + + +def test_playground_url_base(patch_base_class): + stream = YouniumStream(authenticator=None, playground=True) + expected_url_base = "https://apisandbox.younium.com" + assert stream.url_base == expected_url_base + + +def test_use_playground_url_base(patch_base_class): + stream = YouniumStream(authenticator=None, playground=True) + expected_url_base = "https://apisandbox.younium.com" + assert stream.url_base == expected_url_base + + +def test_http_method(patch_base_class): + stream = YouniumStream(authenticator=None) + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = YouniumStream(authenticator=None) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = YouniumStream(authenticator=None) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/sources/younium.md b/docs/integrations/sources/younium.md new file mode 100644 index 000000000000..7aa192d09dff --- /dev/null +++ b/docs/integrations/sources/younium.md @@ -0,0 +1,36 @@ +# Younium + +This page contains the setup guide and reference information for the Younium source connector. + +## Prerequisites + +This Younium source uses the [Younium API](https://developer.younium.com/). + +## Setup guide +### Step 1: Set up Younium + +#### For Airbyte OSS: +1. Navigate to the Airbyte Open Source dashboard +2. Enter a name for your source +3. Enter your Younium `username` +4. Enter your Younium `password` +5. Enter your Younium `legal_entity`. You can find the legal entity name in your account setting if you log in to the [Younium web platform](https://app.younium.com/) +7. Click **Set up source** + +## Supported sync modes + +The Younium source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :--- | :-- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | + +## Supported Streams + +* [Subscriptions](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Subscriptions) +* [Products](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Products) +* [Invoices](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Invoices) \ No newline at end of file