From a6d3c4172e4a41fbaf76cbc085ea38524e4fe08d Mon Sep 17 00:00:00 2001 From: Delena Malan Date: Thu, 27 Oct 2022 11:01:57 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20Workable=20(low-?= =?UTF-8?q?code=20SDK)=20(#18033)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Generate initial Workable connector * Implement jobs route * Make acceptance tests pass * Implement pagination * Try to fix pagination (not yet working) * Make pagination work * Add created_after parameter * Add Workable docs * Move unused test * Add candidates stream * Fix acceptance tests * Remove unused files * Add stages stream * Add recruiters stream * Remove sample_state file * Add Workable to docs/integrations/README.md * Update changelog * Change Sentry page size back * Rename created_after_date to start_date * Remove data type map * Add start_date to sample_config.json * Add streams documentation to yaml * add title to params inspec * format files * fix: added source to source_definitions Co-authored-by: Harshith Mullapudi Co-authored-by: marcosmarxm --- .../src/main/resources/icons/workable.svg | 23 +++++ .../resources/seed/source_definitions.yaml | 7 ++ .../src/main/resources/seed/source_specs.yaml | 34 ++++++++ .../connectors/source-workable/.dockerignore | 6 ++ .../source-workable/.python-version | 1 + .../connectors/source-workable/Dockerfile | 38 +++++++++ .../connectors/source-workable/README.md | 79 +++++++++++++++++ .../connectors/source-workable/__init__.py | 3 + .../acceptance-test-config.yml | 20 +++++ .../source-workable/acceptance-test-docker.sh | 16 ++++ .../connectors/source-workable/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 ++ .../integration_tests/acceptance.py | 14 ++++ .../integration_tests/configured_catalog.json | 22 +++++ .../integration_tests/invalid_config.json | 5 ++ .../integration_tests/sample_config.json | 5 ++ .../connectors/source-workable/main.py | 13 +++ .../source-workable/requirements.txt | 2 + .../connectors/source-workable/setup.py | 29 +++++++ .../source_workable/__init__.py | 8 ++ .../source_workable/schemas/candidates.json | 21 +++++ .../source_workable/schemas/jobs.json | 83 ++++++++++++++++++ .../source_workable/schemas/recruiters.json | 15 ++++ .../source_workable/schemas/stages.json | 18 ++++ .../source-workable/source_workable/source.py | 18 ++++ .../source-workable/source_workable/spec.yaml | 28 +++++++ .../source_workable/workable.yaml | 84 +++++++++++++++++++ docs/integrations/README.md | 1 + docs/integrations/sources/workable.md | 55 ++++++++++++ 30 files changed, 665 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/workable.svg create mode 100644 airbyte-integrations/connectors/source-workable/.dockerignore create mode 100644 airbyte-integrations/connectors/source-workable/.python-version create mode 100644 airbyte-integrations/connectors/source-workable/Dockerfile create mode 100644 airbyte-integrations/connectors/source-workable/README.md create mode 100644 airbyte-integrations/connectors/source-workable/__init__.py create mode 100644 airbyte-integrations/connectors/source-workable/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-workable/build.gradle create mode 100644 airbyte-integrations/connectors/source-workable/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-workable/main.py create mode 100644 airbyte-integrations/connectors/source-workable/requirements.txt create mode 100644 airbyte-integrations/connectors/source-workable/setup.py create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/__init__.py create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/source.py create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/spec.yaml create mode 100644 airbyte-integrations/connectors/source-workable/source_workable/workable.yaml create mode 100644 docs/integrations/sources/workable.md diff --git a/airbyte-config/init/src/main/resources/icons/workable.svg b/airbyte-config/init/src/main/resources/icons/workable.svg new file mode 100644 index 000000000000..10c4da2ac370 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/workable.svg @@ -0,0 +1,23 @@ + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0b5ab9561c40..782770dd3148 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1199,6 +1199,13 @@ icon: woocommerce.svg sourceType: api releaseStage: alpha +- name: Workable + sourceDefinitionId: ef3c99c6-9e90-43c8-9517-926cfd978517 + dockerRepository: airbyte/source-workable + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/workable + sourceType: api + releaseStage: alpha - name: Wrike sourceDefinitionId: 9c13f986-a13b-4988-b808-4705badf71c2 dockerRepository: airbyte/source-wrike diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 37f4ca56b1f4..d6c078e5b864 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -12219,6 +12219,40 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-workable:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/workable" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Workable API Spec" + type: "object" + required: + - "api_key" + - "account_subdomain" + - "start_date" + additionalProperties: true + properties: + api_key: + title: "API Key" + type: "string" + description: "Your Workable API Key. See here." + airbyte_secret: true + account_subdomain: + title: "Account Subdomain" + type: "string" + description: "Your Workable account subdomain, e.g. https://your_account_subdomain.workable.com." + start_date: + title: "Start Date" + type: "string" + description: "Get data that was created since this date (format: YYYYMMDDTHHMMSSZ)." + pattern: "^[0-9]{8}T[0-9]{6}Z$" + examples: + - "20150708T115616Z" + - "20221115T225616Z" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-wrike:0.1.0" spec: documentationUrl: "https://docsurl.com" diff --git a/airbyte-integrations/connectors/source-workable/.dockerignore b/airbyte-integrations/connectors/source-workable/.dockerignore new file mode 100644 index 000000000000..f0aecf092ee4 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_workable +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-workable/.python-version b/airbyte-integrations/connectors/source-workable/.python-version new file mode 100644 index 000000000000..a9f8d1be337f --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/.python-version @@ -0,0 +1 @@ +3.9.11 diff --git a/airbyte-integrations/connectors/source-workable/Dockerfile b/airbyte-integrations/connectors/source-workable/Dockerfile new file mode 100644 index 000000000000..625c27d740a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_workable ./source_workable + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-workable diff --git a/airbyte-integrations/connectors/source-workable/README.md b/airbyte-integrations/connectors/source-workable/README.md new file mode 100644 index 000000000000..88e9aaad1ca1 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/README.md @@ -0,0 +1,79 @@ +# Workable Source + +This is the repository for the Workable configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/workable). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-workable:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/workable) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_workable/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source workable test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-workable:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-workable:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-workable:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workable:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workable:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-workable:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-workable:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-workable:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-workable/__init__.py b/airbyte-integrations/connectors/source-workable/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-workable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-workable/acceptance-test-config.yml new file mode 100644 index 000000000000..bd4e2a1b8f4c --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-workable:dev +tests: + spec: + - spec_path: "source_workable/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-workable/build.gradle b/airbyte-integrations/connectors/source-workable/build.gradle new file mode 100644 index 000000000000..0028e9d2c861 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_workable' +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/__init__.py b/airbyte-integrations/connectors/source-workable/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..c1aa3481db58 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "jobs": { + "id": 0 + } +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4830b6e1848f --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "candidates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json new file mode 100644 index 000000000000..8b389bcec35a --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "account_subdomain": "invalid_subdomain", + "start_date": "00001115T225616Z" +} diff --git a/airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json new file mode 100644 index 000000000000..c04684881f28 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "account_subdomain": "", + "start_date": "" +} diff --git a/airbyte-integrations/connectors/source-workable/main.py b/airbyte-integrations/connectors/source-workable/main.py new file mode 100644 index 000000000000..f52cd2dc1619 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_workable import SourceWorkable + +if __name__ == "__main__": + source = SourceWorkable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-workable/requirements.txt b/airbyte-integrations/connectors/source-workable/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-workable/setup.py b/airbyte-integrations/connectors/source-workable/setup.py new file mode 100644 index 000000000000..257af451405d --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_workable", + description="Source implementation for Workable.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-workable/source_workable/__init__.py b/airbyte-integrations/connectors/source-workable/source_workable/__init__.py new file mode 100644 index 000000000000..36ebfc8b4f25 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWorkable + +__all__ = ["SourceWorkable"] diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json new file mode 100644 index 000000000000..d06a94047edf --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/candidates.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "headline": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json new file mode 100644 index 000000000000..1b02f961a15d --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/jobs.json @@ -0,0 +1,83 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "full_title": { + "type": ["null", "string"] + }, + "shortcode": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "department": { + "type": ["null", "string"] + }, + "department_hierarchy": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + } + } + }, + "minItems": 0 + }, + "url": { + "type": ["null", "string"] + }, + "application_url": { + "type": ["null", "string"] + }, + "shortlink": { + "type": ["null", "string"] + }, + "location": { + "type": "object", + "properties": { + "location_str": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "region_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip_code": { + "type": ["null", "string"] + }, + "telecommuting": { + "type": "boolean" + } + } + }, + "created_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json new file mode 100644 index 000000000000..b776f6583ca0 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/recruiters.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json b/airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json new file mode 100644 index 000000000000..14eb128a8922 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/schemas/stages.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "slug": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "kind": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-workable/source_workable/source.py b/airbyte-integrations/connectors/source-workable/source_workable/source.py new file mode 100644 index 000000000000..cbb89faaacb7 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceWorkable(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "workable.yaml"}) diff --git a/airbyte-integrations/connectors/source-workable/source_workable/spec.yaml b/airbyte-integrations/connectors/source-workable/source_workable/spec.yaml new file mode 100644 index 000000000000..00fd2eb461f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/spec.yaml @@ -0,0 +1,28 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/workable +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Workable API Spec + type: object + required: + - api_key + - account_subdomain + - start_date + additionalProperties: true + properties: + api_key: + title: API Key + type: string + description: Your Workable API Key. See here. + airbyte_secret: true + account_subdomain: + title: Account Subdomain + type: string + description: Your Workable account subdomain, e.g. https://your_account_subdomain.workable.com. + start_date: + title: Start Date + type: string + description: "Get data that was created since this date (format: YYYYMMDDTHHMMSSZ)." + pattern: ^[0-9]{8}T[0-9]{6}Z$ + examples: + - 20150708T115616Z + - 20221115T225616Z diff --git a/airbyte-integrations/connectors/source-workable/source_workable/workable.yaml b/airbyte-integrations/connectors/source-workable/source_workable/workable.yaml new file mode 100644 index 000000000000..00d4a323b89f --- /dev/null +++ b/airbyte-integrations/connectors/source-workable/source_workable/workable.yaml @@ -0,0 +1,84 @@ +version: "0.1.0" + +definitions: + page_size: 100 + schema_loader: + type: JsonSchema + file_path: "./source_sentry/schemas/{{ options.name }}.json" + selector: + extractor: + field_pointer: [] + requester: + url_base: "https://{{ config['account_subdomain'] }}.workable.com" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_key'] }}" + request_options_provider: + request_parameters: + created_after: "{{ config['start_date'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + requester: + $ref: "*ref(definitions.requester)" + paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + limit_option: + inject_into: "request_parameter" + field_name: "" + page_token_option: + inject_into: "path" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response.paging.next }}" + stop_condition: "{{ 'next' not in response['paging'] }}" + page_size: "*ref(definitions.page_size)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + jobs_stream: # https://workable.readme.io/reference/jobs + $ref: "*ref(definitions.base_stream)" + $options: + name: "jobs" + primary_key: "id" + path: "/spi/v3/jobs" + field_pointer: ["jobs"] + candidates_stream: # https://workable.readme.io/reference/job-candidates-index + $ref: "*ref(definitions.base_stream)" + $options: + name: "candidates" + primary_key: "id" + path: "/spi/v3/candidates" + field_pointer: ["candidates"] + stages_stream: # https://workable.readme.io/reference/stages + $ref: "*ref(definitions.base_stream)" + $options: + name: "stages" + primary_key: "slug" + path: "/spi/v3/stages" + field_pointer: ["stages"] + recruiters_stream: # https://workable.readme.io/reference/recruiters + $ref: "*ref(definitions.base_stream)" + $options: + name: "recruiters" + primary_key: "id" + path: "/spi/v3/recruiters" + field_pointer: ["recruiters"] + +streams: + - "*ref(definitions.jobs_stream)" + - "*ref(definitions.candidates_stream)" + - "*ref(definitions.stages_stream)" + - "*ref(definitions.recruiters_stream)" + +check: + stream_names: + - "jobs" + - "candidates" + - "stages" + - "recruiters" diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 26454b792a87..578fd492e152 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -173,6 +173,7 @@ For more information about the grading system, see [Product Release Stages](http | [Whisky Hunter](sources/whisky-hunter.md ) | Alpha | No | | [WooCommerce](sources/woocommerce.md) | Alpha | No | | [Wordpress](sources/wordpress.md) | Alpha | No | +| [Workable](sources/workable.md) | Alpha | No | | [Wrike](sources/wrike.md) | Alpha | No | | [YouTube Analytics](sources/youtube-analytics.md) | Beta | Yes | | [Xkcd](sources/xkcd.md) | Alpha | No | diff --git a/docs/integrations/sources/workable.md b/docs/integrations/sources/workable.md new file mode 100644 index 000000000000..de9267036cfa --- /dev/null +++ b/docs/integrations/sources/workable.md @@ -0,0 +1,55 @@ +# Workable + +This page guides you through the process of setting up the Workable source connector. + +## Prerequisites + +You can find or create a Workable access token within the [Workable Integrations Settings page](https://test-432879.workable.com/backend/settings/integrations). See [this page](https://workable.readme.io/reference/generate-an-access-token#generate-an-api-access-token) for a step-by-step guide. + +## Setup guide + +## Step 1: Set up the Workable connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the Workable connector and select **Workable** from the Source type dropdown. +4. Enter your `api_token` - Workable Access Token. +5. Enter your `account_subdomain` - Sub-domain for your organization on Workable, e.g. https://YOUR_ACCOUNT_SUBDOMAIN.workable.com. +6. Enter your `created_after_date` - The earliest created at date from which you want to sync your Workable data. +7. Click **Set up source**. + +### For Airbyte OSS: + +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +4. Enter your `api_token` - Workable Access Token. +5. Enter your `account_subdomain` - Sub-domain for your organization on Workable, e.g. https://YOUR_ACCOUNT_SUBDOMAIN.workable.com. +6. Enter your `created_after_date` - The earliest created at date from which you want to sync your Workable data. +7. Click **Set up source**. + +## Supported sync modes + +The Workable source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | Yes | +| Namespaces | No | + +## Supported Streams + +* [Jobs](https://workable.readme.io/reference/jobs) +* [Candidates](https://workable.readme.io/reference/job-candidates-index) +* [Stages](https://workable.readme.io/reference/stages) +* [Recruiters](https://workable.readme.io/reference/recruiters) + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------| +| 0.1.0 | 2022-10-15 | [18033](https://github.com/airbytehq/airbyte/pull/18033) | New Source: Workable | +