From f34699e111b458114e11f7b1c48b46448a57291b Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Thu, 13 Oct 2022 18:50:05 +0100 Subject: [PATCH 01/13] Initial commit --- .../source-alpha-vantage/.dockerignore | 6 + .../source-alpha-vantage/Dockerfile | 38 ++++++ .../connectors/source-alpha-vantage/README.md | 79 +++++++++++++ .../source-alpha-vantage/__init__.py | 3 + .../acceptance-test-config.yml | 30 +++++ .../acceptance-test-docker.sh | 16 +++ .../source-alpha-vantage/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 +++ .../integration_tests/catalog.json | 39 +++++++ .../integration_tests/configured_catalog.json | 22 ++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-alpha-vantage/main.py | 13 +++ .../source-alpha-vantage/requirements.txt | 2 + .../connectors/source-alpha-vantage/setup.py | 29 +++++ .../source_alpha_vantage/__init__.py | 8 ++ .../source_alpha_vantage/alpha_vantage.yaml | 109 ++++++++++++++++++ .../source_alpha_vantage/schemas/TODO.md | 16 +++ .../schemas/customers.json | 16 +++ .../schemas/employees.json | 19 +++ .../source_alpha_vantage/source.py | 18 +++ .../source_alpha_vantage/spec.yaml | 53 +++++++++ 25 files changed, 560 insertions(+) create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/.dockerignore create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/Dockerfile create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/README.md create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/__init__.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/build.gradle create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/main.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/requirements.txt create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/setup.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/__init__.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml diff --git a/airbyte-integrations/connectors/source-alpha-vantage/.dockerignore b/airbyte-integrations/connectors/source-alpha-vantage/.dockerignore new file mode 100644 index 000000000000..5a1cff6bff4a --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_alpha_vantage +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-alpha-vantage/Dockerfile b/airbyte-integrations/connectors/source-alpha-vantage/Dockerfile new file mode 100644 index 000000000000..7735d2dd4e93 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_alpha_vantage ./source_alpha_vantage + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-alpha-vantage diff --git a/airbyte-integrations/connectors/source-alpha-vantage/README.md b/airbyte-integrations/connectors/source-alpha-vantage/README.md new file mode 100644 index 000000000000..bb054c73dec7 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/README.md @@ -0,0 +1,79 @@ +# Alpha Vantage Source + +This is the repository for the Alpha Vantage configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/alpha-vantage). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-alpha-vantage:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/alpha-vantage) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_alpha_vantage/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source alpha-vantage test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-alpha-vantage:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-alpha-vantage:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-alpha-vantage:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-alpha-vantage:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-alpha-vantage:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-alpha-vantage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-alpha-vantage:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-alpha-vantage:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-alpha-vantage/__init__.py b/airbyte-integrations/connectors/source-alpha-vantage/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml new file mode 100644 index 000000000000..ac0d4adadf08 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-alpha-vantage:dev +tests: + spec: + - spec_path: "source_alpha_vantage/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.txt" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-alpha-vantage/build.gradle b/airbyte-integrations/connectors/source-alpha-vantage/build.gradle new file mode 100644 index 000000000000..f8d1a7d118a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_alpha_vantage' +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/__init__.py b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/main.py b/airbyte-integrations/connectors/source-alpha-vantage/main.py new file mode 100644 index 000000000000..c4d96e59a041 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_alpha_vantage import SourceAlphaVantage + +if __name__ == "__main__": + source = SourceAlphaVantage() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt b/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-alpha-vantage/setup.py b/airbyte-integrations/connectors/source-alpha-vantage/setup.py new file mode 100644 index 000000000000..034b1947464c --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_alpha_vantage", + description="Source implementation for Alpha Vantage.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/__init__.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/__init__.py new file mode 100644 index 000000000000..086d696c81dd --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAlphaVantage + +__all__ = ["SourceAlphaVantage"] diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml new file mode 100644 index 000000000000..cca43212cf49 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml @@ -0,0 +1,109 @@ +version: "0.1.0" + +definitions: + schema_loader: + type: JsonSchema + file_path: "./source_alpha_vantage/schemas/{{ options['name'] }}.json" + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_pointer: + - "{{ options['dpath'] }}" + requester: + url_base: "https://www.alphavantage.co" + http_method: "GET" + authenticator: + type: NoAuth + request_options_provider: + request_parameters: + apikey: "{{ config['api_key'] }}" + symbol: "{{ config['symbol'] }}" +# function: "{{ options['function'] }}" + function: GLOBAL_QUOTE + datatype: json + outputsize: "{{ config['outputsize'] }}" + interval: "{{ config['interval'] }}" + adjusted: "{{ config['adjusted'] }}" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + time_series_intraday_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_intraday" + dpath: "Time Series ({{ config['interval'] }})" + function: "TIME_SERIES_INTRADAY" + path: "/query" + time_series_daily_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_daily" + dpath: "Time Series (Daily)" + function: "TIME_SERIES_DAILY" + path: "/query" + time_series_daily_adjusted_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_daily_adjusted" + dpath: "Time Series (Daily)" + function: "TIME_SERIES_DAILY_ADJUSTED" + path: "/query" + time_series_weekly_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_weekly" + dpath: "Weekly Time Series" + function: "TIME_SERIES_WEEKLY" + path: "/query" + time_series_weekly_adjusted_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_weekly_adjusted" + dpath: "Weekly Adjusted Time Series" + function: "TIME_SERIES_WEEKLY_ADJUSTED" + path: "/query" + time_series_monthly_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_monthly" + dpath: "Weekly Time Series" + function: "TIME_SERIES_MONTHLY" + path: "/query" + time_series_monthly_adjusted_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "time_series_monthly_adjusted" + dpath: "Weekly Adjusted Time Series" + function: "TIME_SERIES_MONTHLY_ADJUSTED" + path: "/query" + quote_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "quote" + dpath: "Global Quote" + function: "GLOBAL_QUOTE" + path: "/query" + +streams: + - "*ref(definitions.time_series_intraday_stream)" + - "*ref(definitions.time_series_daily_stream)" + - "*ref(definitions.time_series_daily_adjusted_stream)" + - "*ref(definitions.time_series_weekly_stream)" + - "*ref(definitions.time_series_weekly_adjusted_stream)" + - "*ref(definitions.time_series_monthly_stream)" + - "*ref(definitions.time_series_monthly_adjusted_stream)" + - "*ref(definitions.quote_stream)" + +check: + stream_names: +# - "time_series_weekly" +# - "time_series_weekly_adjusted" + - "quote" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md new file mode 100644 index 000000000000..7ce916b57165 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md @@ -0,0 +1,16 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +You can describe the schema of your streams using one `.json` file per stream. + +## Static schemas +From the `alpha_vantage.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: +``` +schema_loader: + type: JsonSchema + file_path: "./source_alpha_vantage/schemas/customers.json" +``` +Every stream specified in the configuration file should have a corresponding `.json` schema file. + +Delete this file once you're done. Or don't. Up to you :) + diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py new file mode 100644 index 000000000000..b8e22cf0db5d --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceAlphaVantage(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "alpha_vantage.yaml"}) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml new file mode 100644 index 000000000000..f48e1f05eeef --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml @@ -0,0 +1,53 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Alpha Vantage Spec + type: object + required: + - api_key + - symbol + properties: + api_key: + title: API Key + type: string + description: API Key + airbyte_secret: true + order: 0 + symbol: + title: Symbol + type: string + description: Stock symbol (with exchange code) + examples: + - AAPL + - TSCO.LON + order: 1 + interval: + title: Interval + type: string + description: | + Time-series data point interval. Required for intraday endpoints. + enum: + - 1min + - 5min + - 15min + - 30min + - 60min + default: 1min + order: 2 + adjusted: + title: Adjusted? + type: boolean + description: | + Whether to return adjusted data. Only applicable to intraday endpoints. + default: false + order: 4 + outputsize: + title: Output Size + type: string + description: | + Whether to return full or compact data (the last 100 data points). + enum: + - compact + - full + default: compact + order: 5 From 48df82e3a7bb7147f6cddc8b99f7097b6fa64ea1 Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Fri, 21 Oct 2022 21:54:58 +0100 Subject: [PATCH 02/13] Tidy and add custom dpath extractor --- .../integration_tests/catalog.json | 76 +++++++++------ .../integration_tests/configured_catalog.json | 19 +++- .../source_alpha_vantage/alpha_vantage.yaml | 30 +++--- .../object_dpath_extractor.py | 83 ++++++++++++++++ .../source_alpha_vantage/schemas/TODO.md | 16 ---- .../schemas/customers.json | 16 ---- .../schemas/employees.json | 19 ---- .../schemas/time_series_daily.json | 21 ++++ .../schemas/time_series_daily_adjusted.json | 30 ++++++ .../schemas/time_series_intraday.json | 21 ++++ .../schemas/time_series_monthly.json | 21 ++++ .../schemas/time_series_monthly_adjusted.json | 27 ++++++ .../schemas/time_series_weekly.json | 21 ++++ .../schemas/time_series_weekly_adjusted.json | 27 ++++++ .../unit_tests/__init__.py | 0 .../unit_tests/test_object_dpath_extractor.py | 96 +++++++++++++++++++ 16 files changed, 420 insertions(+), 103 deletions(-) create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py delete mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md delete mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json delete mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily_adjusted.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_intraday.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly_adjusted.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly_adjusted.json create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json index 6799946a6851..a3ef2ace2f43 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json @@ -1,39 +1,53 @@ { "streams": [ { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "time_series_intraday", + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } + "default_cursor_field": "timestamp", + "json_schema": {} }, { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } + "name": "time_series_daily", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + }, + { + "name": "time_series_daily_adjusted", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + }, + { + "name": "time_series_weekly", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + }, + { + "name": "time_series_weekly_adjusted", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + }, + { + "name": "time_series_monthly", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + }, + { + "name": "time_series_monthly_adjusted", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": "date", + "json_schema": {} + }} ] } diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json index 36f0468db0d8..dfafcb1e168f 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/configured_catalog.json @@ -2,7 +2,7 @@ "streams": [ { "stream": { - "name": "customers", + "name": "time_series_daily", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -11,12 +11,21 @@ }, { "stream": { - "name": "employees", + "name": "time_series_weekly", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "time_series_monthly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml index cca43212cf49..027e51ef11f3 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/alpha_vantage.yaml @@ -7,9 +7,10 @@ definitions: selector: type: RecordSelector extractor: - type: DpathExtractor + class_name: source_alpha_vantage.object_dpath_extractor.ObjectDpathExtractor field_pointer: - "{{ options['dpath'] }}" + inject_key_as_field: "{{ options['key_field'] }}" requester: url_base: "https://www.alphavantage.co" http_method: "GET" @@ -19,8 +20,7 @@ definitions: request_parameters: apikey: "{{ config['api_key'] }}" symbol: "{{ config['symbol'] }}" -# function: "{{ options['function'] }}" - function: GLOBAL_QUOTE + function: "{{ options['function'] }}" datatype: json outputsize: "{{ config['outputsize'] }}" interval: "{{ config['interval'] }}" @@ -42,6 +42,7 @@ definitions: dpath: "Time Series ({{ config['interval'] }})" function: "TIME_SERIES_INTRADAY" path: "/query" + key_field: "timestamp" time_series_daily_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -49,6 +50,7 @@ definitions: dpath: "Time Series (Daily)" function: "TIME_SERIES_DAILY" path: "/query" + key_field: "date" time_series_daily_adjusted_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -56,6 +58,7 @@ definitions: dpath: "Time Series (Daily)" function: "TIME_SERIES_DAILY_ADJUSTED" path: "/query" + key_field: "date" time_series_weekly_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -63,6 +66,7 @@ definitions: dpath: "Weekly Time Series" function: "TIME_SERIES_WEEKLY" path: "/query" + key_field: "date" time_series_weekly_adjusted_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -70,27 +74,23 @@ definitions: dpath: "Weekly Adjusted Time Series" function: "TIME_SERIES_WEEKLY_ADJUSTED" path: "/query" + key_field: "date" time_series_monthly_stream: $ref: "*ref(definitions.base_stream)" $options: name: "time_series_monthly" - dpath: "Weekly Time Series" + dpath: "Monthly Time Series" function: "TIME_SERIES_MONTHLY" path: "/query" + key_field: "date" time_series_monthly_adjusted_stream: $ref: "*ref(definitions.base_stream)" $options: name: "time_series_monthly_adjusted" - dpath: "Weekly Adjusted Time Series" + dpath: "Monthly Adjusted Time Series" function: "TIME_SERIES_MONTHLY_ADJUSTED" path: "/query" - quote_stream: - $ref: "*ref(definitions.base_stream)" - $options: - name: "quote" - dpath: "Global Quote" - function: "GLOBAL_QUOTE" - path: "/query" + key_field: "date" streams: - "*ref(definitions.time_series_intraday_stream)" @@ -100,10 +100,8 @@ streams: - "*ref(definitions.time_series_weekly_adjusted_stream)" - "*ref(definitions.time_series_monthly_stream)" - "*ref(definitions.time_series_monthly_adjusted_stream)" - - "*ref(definitions.quote_stream)" check: stream_names: -# - "time_series_weekly" -# - "time_series_weekly_adjusted" - - "quote" + - "time_series_weekly" + - "time_series_weekly_adjusted" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py new file mode 100644 index 000000000000..dd2728ffdf48 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py @@ -0,0 +1,83 @@ +from dataclasses import dataclass +from typing import Mapping, Any + +import dpath.util +import requests +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class ObjectDpathExtractor(DpathExtractor): + """ + Record extractor that searches a decoded response over a path defined as an array of fields. + + Extends the DpathExtractor to allow for a list of records to be generated from a dpath that points + to an object, where the object's values are individual records. + + Example data: + ``` + { + "data": { + "2022-01-01": { + "id": "id1", + "name": "name1", + ... + }, + "2022-01-02": { + "id": "id2", + "name": "name2", + ... + }, + ... + } + ``` + + There is an optional `inject_key_as_field` parameter that can be used to inject the key of the object + as a field in the record. For example, if `inject_key_as_field` is set to `date`, the above data would + be transformed to: + ``` + [ + { + "date": "2022-01-01", + "id": "id1", + "name": "name1", + ... + }, + { + "date": "2022-01-02", + "id": "id2", + "name": "name2", + ... + }, + ... + ] + """ + inject_key_as_field: str | InterpolatedString | None = None + + def __post_init__(self, options: Mapping[str, Any]): + self.inject_key_as_field = InterpolatedString.create(self.inject_key_as_field, options=options) + for pointer_index in range(len(self.field_pointer)): + if isinstance(self.field_pointer[pointer_index], str): + self.field_pointer[pointer_index] = InterpolatedString.create(self.field_pointer[pointer_index], options=options) + + def extract_records(self, response: requests.Response) -> list[Record]: + response_body = self.decoder.decode(response) + if len(self.field_pointer) == 0: + extracted = response_body + else: + pointer = [pointer.eval(self.config) for pointer in self.field_pointer] + extracted = dpath.util.get(response_body, pointer, default=[]) + if isinstance(extracted, list): + return extracted + elif isinstance(extracted, dict) and isinstance(extracted[list(extracted.keys())[0]], dict): # Ensure object is dict[str, dict] + if not self.inject_key_as_field: + return [value for _, value in extracted.items()] + else: + key_field = self.inject_key_as_field.eval(self.config) + return [{key_field: key, **value} for key, value in extracted.items()] + elif extracted: + return [extracted] + else: + return [] diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md deleted file mode 100644 index 7ce916b57165..000000000000 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/TODO.md +++ /dev/null @@ -1,16 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -You can describe the schema of your streams using one `.json` file per stream. - -## Static schemas -From the `alpha_vantage.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: -``` -schema_loader: - type: JsonSchema - file_path: "./source_alpha_vantage/schemas/customers.json" -``` -Every stream specified in the configuration file should have a corresponding `.json` schema file. - -Delete this file once you're done. Or don't. Up to you :) - diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json deleted file mode 100644 index 9a4b13485836..000000000000 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/customers.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json deleted file mode 100644 index 2fa01a0fa1ff..000000000000 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/employees.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily.json new file mode 100644 index 000000000000..666425ebf3dd --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": ["string", "null"] + }, + "2. high": { + "type": ["string", "null"] + }, + "3. low": { + "type": ["string", "null"] + }, + "4. close": { + "type": ["string", "null"] + }, + "5. volume": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily_adjusted.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily_adjusted.json new file mode 100644 index 000000000000..49f6c50c55eb --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_daily_adjusted.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": "string" + }, + "2. high": { + "type": "string" + }, + "3. low": { + "type": "string" + }, + "4. close": { + "type": "string" + }, + "5. adjusted close": { + "type": "string" + }, + "6. volume": { + "type": "string" + }, + "7. dividend amount": { + "type": "string" + }, + "8. split coefficient": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_intraday.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_intraday.json new file mode 100644 index 000000000000..666425ebf3dd --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_intraday.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": ["string", "null"] + }, + "2. high": { + "type": ["string", "null"] + }, + "3. low": { + "type": ["string", "null"] + }, + "4. close": { + "type": ["string", "null"] + }, + "5. volume": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly.json new file mode 100644 index 000000000000..666425ebf3dd --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": ["string", "null"] + }, + "2. high": { + "type": ["string", "null"] + }, + "3. low": { + "type": ["string", "null"] + }, + "4. close": { + "type": ["string", "null"] + }, + "5. volume": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly_adjusted.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly_adjusted.json new file mode 100644 index 000000000000..962674bfc903 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_monthly_adjusted.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": "string" + }, + "2. high": { + "type": "string" + }, + "3. low": { + "type": "string" + }, + "4. close": { + "type": "string" + }, + "5. adjusted close": { + "type": "string" + }, + "6. volume": { + "type": "string" + }, + "7. dividend amount": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly.json new file mode 100644 index 000000000000..666425ebf3dd --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": ["string", "null"] + }, + "2. high": { + "type": ["string", "null"] + }, + "3. low": { + "type": ["string", "null"] + }, + "4. close": { + "type": ["string", "null"] + }, + "5. volume": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly_adjusted.json b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly_adjusted.json new file mode 100644 index 000000000000..962674bfc903 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/schemas/time_series_weekly_adjusted.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "1. open": { + "type": "string" + }, + "2. high": { + "type": "string" + }, + "3. low": { + "type": "string" + }, + "4. close": { + "type": "string" + }, + "5. adjusted close": { + "type": "string" + }, + "6. volume": { + "type": "string" + }, + "7. dividend amount": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/__init__.py b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py new file mode 100644 index 000000000000..21568d72a082 --- /dev/null +++ b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py @@ -0,0 +1,96 @@ +import json + +from requests import Response + +from source_alpha_vantage.object_dpath_extractor import ObjectDpathExtractor + + +def _create_response() -> Response: + response_body = { + "data": { + "2022-01-01": { + "id": "id1", + "name": "name1", + }, + "2022-01-02": { + "id": "id2", + "name": "name2", + }, + } + } + + response = Response() + response._content = json.dumps(response_body).encode("utf-8") + return response + + +def test_no_key_injection(): + extractor = ObjectDpathExtractor( + field_pointer=["data"], + config={}, + options={}, + ) + + response = _create_response() + records = extractor.extract_records(response) + + assert records == [ + { + "id": "id1", + "name": "name1", + }, + { + "id": "id2", + "name": "name2", + }, + ] + + +def test_key_injection(): + extractor = ObjectDpathExtractor( + field_pointer=["data"], + inject_key_as_field="date", + config={}, + options={}, + ) + + response = _create_response() + records = extractor.extract_records(response) + + assert records == [ + { + "date": "2022-01-01", + "id": "id1", + "name": "name1", + }, + { + "date": "2022-01-02", + "id": "id2", + "name": "name2", + }, + ] + + +def test_key_injection_with_interpolation(): + extractor = ObjectDpathExtractor( + field_pointer=["data"], + inject_key_as_field="{{ config['key_field'] }}", + config={"key_field": "date"}, + options={}, + ) + + response = _create_response() + records = extractor.extract_records(response) + + assert records == [ + { + "date": "2022-01-01", + "id": "id1", + "name": "name1", + }, + { + "date": "2022-01-02", + "id": "id2", + "name": "name2", + }, + ] From 804efe2fc7735f901bb1e930f819480027b99e2c Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Fri, 21 Oct 2022 21:59:00 +0100 Subject: [PATCH 03/13] Get acceptance tests passing --- .../source-alpha-vantage/acceptance-test-config.yml | 10 ---------- .../source_alpha_vantage/object_dpath_extractor.py | 4 ++-- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml index ac0d4adadf08..e96e42b295ab 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml @@ -15,16 +15,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.txt" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py index dd2728ffdf48..8f8a1d13a2ac 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Mapping, Any +from typing import Mapping, Any, Union import dpath.util import requests @@ -54,7 +54,7 @@ class ObjectDpathExtractor(DpathExtractor): ... ] """ - inject_key_as_field: str | InterpolatedString | None = None + inject_key_as_field: Union[str, InterpolatedString] = None def __post_init__(self, options: Mapping[str, Any]): self.inject_key_as_field = InterpolatedString.create(self.inject_key_as_field, options=options) From d364eeac79eb01c06880277d5255dee545e58969 Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Sun, 23 Oct 2022 10:34:56 +0100 Subject: [PATCH 04/13] Improve dict-checking for ObjectDpathExtractor --- .../source_alpha_vantage/object_dpath_extractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py index 8f8a1d13a2ac..8413a9369926 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py @@ -71,7 +71,7 @@ def extract_records(self, response: requests.Response) -> list[Record]: extracted = dpath.util.get(response_body, pointer, default=[]) if isinstance(extracted, list): return extracted - elif isinstance(extracted, dict) and isinstance(extracted[list(extracted.keys())[0]], dict): # Ensure object is dict[str, dict] + elif isinstance(extracted, dict) and all(isinstance(v, dict) for v in extracted.values()): # Ensure object is dict[Hashable, dict] if not self.inject_key_as_field: return [value for _, value in extracted.items()] else: From 6aca05fc2c855d059d715709cc08c3f0ba4ceef8 Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Sun, 23 Oct 2022 10:37:54 +0100 Subject: [PATCH 05/13] Improve dict-checking for ObjectDpathExtractor --- .../unit_tests/test_object_dpath_extractor.py | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py index 21568d72a082..ebe806c01f2a 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py @@ -1,11 +1,18 @@ import json +from typing import Any from requests import Response from source_alpha_vantage.object_dpath_extractor import ObjectDpathExtractor -def _create_response() -> Response: +def _create_response_with_body(body: Any) -> Response: + response = Response() + response._content = json.dumps(body).encode("utf-8") + return response + + +def _create_response_with_dict_of_records() -> Response: response_body = { "data": { "2022-01-01": { @@ -18,10 +25,23 @@ def _create_response() -> Response: }, } } + return _create_response_with_body(response_body) - response = Response() - response._content = json.dumps(response_body).encode("utf-8") - return response + +def _create_response_with_list_of_records() -> Response: + response_body = { + "data": [ + { + "id": "id1", + "name": "name1", + }, + { + "id": "id2", + "name": "name2", + }, + ] + } + return _create_response_with_body(response_body) def test_no_key_injection(): @@ -31,7 +51,7 @@ def test_no_key_injection(): options={}, ) - response = _create_response() + response = _create_response_with_dict_of_records() records = extractor.extract_records(response) assert records == [ @@ -54,7 +74,7 @@ def test_key_injection(): options={}, ) - response = _create_response() + response = _create_response_with_dict_of_records() records = extractor.extract_records(response) assert records == [ @@ -79,7 +99,7 @@ def test_key_injection_with_interpolation(): options={}, ) - response = _create_response() + response = _create_response_with_dict_of_records() records = extractor.extract_records(response) assert records == [ @@ -94,3 +114,25 @@ def test_key_injection_with_interpolation(): "name": "name2", }, ] + + +def test_list_of_records(): + extractor = ObjectDpathExtractor( + field_pointer=["data"], + config={}, + options={}, + ) + + response = _create_response_with_dict_of_records() + records = extractor.extract_records(response) + + assert records == [ + { + "id": "id1", + "name": "name1", + }, + { + "id": "id2", + "name": "name2", + }, + ] From 7c3092ecb0de626711a10b8c5bb66501a2d9c482 Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Sun, 23 Oct 2022 10:40:57 +0100 Subject: [PATCH 06/13] Add more tests to ObjectDpathExtractor --- .../unit_tests/test_object_dpath_extractor.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py index ebe806c01f2a..63398ef4f41d 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py @@ -136,3 +136,34 @@ def test_list_of_records(): "name": "name2", }, ] + + +def test_no_records(): + extractor = ObjectDpathExtractor( + field_pointer=["data"], + config={}, + options={}, + ) + + obj_response = _create_response_with_body({"data": {}}) + obj_records = extractor.extract_records(obj_response) + + assert obj_records == [] + + list_response = _create_response_with_body({"data": []}) + list_records = extractor.extract_records(list_response) + + assert list_records == [] + + +def test_single_record(): + extractor = ObjectDpathExtractor( + field_pointer=["data"], + config={}, + options={}, + ) + + response = _create_response_with_body({"data": {"id": "id1", "name": "name1"}}) + records = extractor.extract_records(response) + + assert records == [{"id": "id1", "name": "name1"}] From e28a43b1be5228d2176fcbd3e4e50fea4bdd8944 Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Tue, 25 Oct 2022 09:43:03 +0100 Subject: [PATCH 07/13] Add docs --- docs/integrations/sources/alpha-vantage.md | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 docs/integrations/sources/alpha-vantage.md diff --git a/docs/integrations/sources/alpha-vantage.md b/docs/integrations/sources/alpha-vantage.md new file mode 100644 index 000000000000..4e7cf2faad83 --- /dev/null +++ b/docs/integrations/sources/alpha-vantage.md @@ -0,0 +1,59 @@ +# Alpha Vantage + +## Sync overview + +This source retrieves time series data from the free +[Alpha Vantage](https://www.alphavantage.co/) API. It supports intraday, daily, +weekly and monthly time series data. + + +### Output schema + +This source is capable of syncing the following streams: + +* `time_series_intraday` +* `time_series_daily` +* `time_series_daily_adjusted` (premium only) +* `time_series_weekly` +* `time_series_weekly_adjusted` +* `time_series_monthly` +* `time_series_monthly_adjusted` + +### Features + +| Feature | Supported? \(Yes/No\) | Notes | +|:------------------|:----------------------|:--------------------------------------------------------------------------------------------------| +| Full Refresh Sync | Yes | | +| Incremental Sync | No | Alpha Vantage always returns the full history, so `Full Refresh` with `Overwrite` is recommended. | +| API Environments | Yes | Both sandbox and production environments are supported. | + +### Performance considerations + +Since a single API call returns the full history of a time series, it is +recommended to use `Full Refresh` with `Overwrite` to avoid storing duplicate +data. + +Also, the data returned can be quite large. + +## Getting started + +### Requirements + +1. Obtain an API key from [Alpha Vantage](https://www.alphavantage.co/support/#api-key). + +### Setup guide + +The following fields are required fields for the connector to work: + +- `api_key`: Your Alpha Vantage API key. +- `symbol`: The symbol of the time series to retrieve, with exchange code if + applicable. For example, `MSFT` or `TSCO.LON`. +- (optional) `interval`: The time-series data point interval. Defaults to 1 minute. +- (optional) `Adjusted?`: Whether the `intraday` endpoint should return adjusted + data. Defaults to `false`. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------| +| 0.1.0 | 2022-10-21 | [18320](https://github.com/airbytehq/airbyte/pull/18320) | New source | From 6db283d32cbc7d83c1c794acb47bc7083cf6f6aa Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Tue, 25 Oct 2022 09:46:13 +0100 Subject: [PATCH 08/13] Address comments --- .../integration_tests/catalog.json | 53 ------------------- .../source_alpha_vantage/spec.yaml | 8 +-- 2 files changed, 4 insertions(+), 57 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json deleted file mode 100644 index a3ef2ace2f43..000000000000 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/catalog.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "streams": [ - { - "name": "time_series_intraday", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "timestamp", - "json_schema": {} - }, - { - "name": "time_series_daily", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "date", - "json_schema": {} - }, - { - "name": "time_series_daily_adjusted", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "date", - "json_schema": {} - }, - { - "name": "time_series_weekly", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "date", - "json_schema": {} - }, - { - "name": "time_series_weekly_adjusted", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "date", - "json_schema": {} - }, - { - "name": "time_series_monthly", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "date", - "json_schema": {} - }, - { - "name": "time_series_monthly_adjusted", - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": "date", - "json_schema": {} - }} - ] -} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml index f48e1f05eeef..cc8a1401b28f 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://docs.airbyte.io/integrations/sources/alpha-vantage connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Alpha Vantage Spec @@ -45,9 +45,9 @@ connectionSpecification: title: Output Size type: string description: | - Whether to return full or compact data (the last 100 data points). + Whether to return full or compact data (the last 100 data points). enum: - - compact - - full + - compact + - full default: compact order: 5 From bcbe89571c67efd1e386d445b65b580edbeddd5b Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Tue, 25 Oct 2022 09:47:34 +0100 Subject: [PATCH 09/13] Update docs --- docs/integrations/sources/alpha-vantage.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/integrations/sources/alpha-vantage.md b/docs/integrations/sources/alpha-vantage.md index 4e7cf2faad83..17dc5daea00f 100644 --- a/docs/integrations/sources/alpha-vantage.md +++ b/docs/integrations/sources/alpha-vantage.md @@ -21,17 +21,17 @@ This source is capable of syncing the following streams: ### Features -| Feature | Supported? \(Yes/No\) | Notes | -|:------------------|:----------------------|:--------------------------------------------------------------------------------------------------| -| Full Refresh Sync | Yes | | -| Incremental Sync | No | Alpha Vantage always returns the full history, so `Full Refresh` with `Overwrite` is recommended. | -| API Environments | Yes | Both sandbox and production environments are supported. | +| Feature | Supported? \(Yes/No\) | Notes | +|:------------------|:----------------------|:--------------------------------------------------------| +| Full Refresh Sync | Yes | | +| Incremental Sync | No | | +| API Environments | Yes | Both sandbox and production environments are supported. | ### Performance considerations -Since a single API call returns the full history of a time series, it is -recommended to use `Full Refresh` with `Overwrite` to avoid storing duplicate -data. +Since a single API call returns the full history of a time series if +configured, it is recommended to use `Full Refresh` with `Overwrite` to avoid +storing duplicate data. Also, the data returned can be quite large. @@ -51,6 +51,9 @@ The following fields are required fields for the connector to work: - (optional) `interval`: The time-series data point interval. Defaults to 1 minute. - (optional) `Adjusted?`: Whether the `intraday` endpoint should return adjusted data. Defaults to `false`. +- (optional) `outputsize`: The size of the time series to retrieve. Defaults to + `compact`, which returns the last 100 data points. `full` returns the full + history. ## Changelog From d3240c28b97f9e35e600352ad028552a249d5b2a Mon Sep 17 00:00:00 2001 From: Isaac Harris-Holt Date: Wed, 26 Oct 2022 08:48:57 +0100 Subject: [PATCH 10/13] Add sample and invalid configs --- .../integration_tests/invalid_config.json | 7 ++++++- .../integration_tests/sample_config.json | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json index f3732995784f..b8e7d27bcb7d 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/invalid_config.json @@ -1,3 +1,8 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" + "api_key": "INVALID_KEY", + "symbol": "FAKE_SYMBOL", + "interval": "60min", + "outputsize": "compact", + "slice": "incorrect_slice", + "adjusted": false } diff --git a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json index ecc4913b84c7..1f1be557c956 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-alpha-vantage/integration_tests/sample_config.json @@ -1,3 +1,8 @@ { - "fix-me": "TODO" + "api_key": "YOUR_API_KEY", + "symbol": "TSLA", + "interval": "60min", + "outputsize": "compact", + "slice": "year1month1", + "adjusted": false } From 61d67389a8b2673d0d4810b3f9af6e12e6d15563 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 31 Oct 2022 22:31:30 -0300 Subject: [PATCH 11/13] format --- .../source-alpha-vantage/acceptance-test-config.yml | 3 ++- .../connectors/source-alpha-vantage/setup.py | 2 +- .../source_alpha_vantage/object_dpath_extractor.py | 7 ++++++- .../unit_tests/test_object_dpath_extractor.py | 5 ++++- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml index e96e42b295ab..daf327f8c10e 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml @@ -14,7 +14,8 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: + - "time_series_monthly" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/setup.py b/airbyte-integrations/connectors/source-alpha-vantage/setup.py index 034b1947464c..5bc457370080 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/setup.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.4", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py index 8413a9369926..1ee461ba13a3 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/source_alpha_vantage/object_dpath_extractor.py @@ -1,5 +1,9 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + from dataclasses import dataclass -from typing import Mapping, Any, Union +from typing import Any, Mapping, Union import dpath.util import requests @@ -54,6 +58,7 @@ class ObjectDpathExtractor(DpathExtractor): ... ] """ + inject_key_as_field: Union[str, InterpolatedString] = None def __post_init__(self, options: Mapping[str, Any]): diff --git a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py index 63398ef4f41d..3324ed175a7b 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py +++ b/airbyte-integrations/connectors/source-alpha-vantage/unit_tests/test_object_dpath_extractor.py @@ -1,8 +1,11 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + import json from typing import Any from requests import Response - from source_alpha_vantage.object_dpath_extractor import ObjectDpathExtractor From 6fd936bdc88e545eef415047abbabefd774f5198 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Tue, 1 Nov 2022 10:31:50 -0300 Subject: [PATCH 12/13] add alpha vantage to source def --- .../init/src/main/resources/seed/source_definitions.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 75aadbd817d0..aa3eb000acaa 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -14,6 +14,13 @@ icon: airtable.svg sourceType: api releaseStage: alpha +- name: Alpha Vantage + sourceDefinitionId: db385323-9333-4fec-bec3-9e0ca9326c90 + dockerRepository: airbyte/source-alpha-vantage + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/alpha-vantage + sourceType: api + releaseStage: alpha - name: AlloyDB for PostgreSQL sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 dockerRepository: airbyte/source-alloydb From 6ac1a3d4f2f4a47c6545540ba1f50ae3281de700 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 1 Nov 2022 16:49:23 +0000 Subject: [PATCH 13/13] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 71503972f7cc..2053c73eda01 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -198,6 +198,63 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-alpha-vantage:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/alpha-vantage" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Alpha Vantage Spec" + type: "object" + required: + - "api_key" + - "symbol" + properties: + api_key: + title: "API Key" + type: "string" + description: "API Key" + airbyte_secret: true + order: 0 + symbol: + title: "Symbol" + type: "string" + description: "Stock symbol (with exchange code)" + examples: + - "AAPL" + - "TSCO.LON" + order: 1 + interval: + title: "Interval" + type: "string" + description: "Time-series data point interval. Required for intraday endpoints.\n" + enum: + - "1min" + - "5min" + - "15min" + - "30min" + - "60min" + default: "1min" + order: 2 + adjusted: + title: "Adjusted?" + type: "boolean" + description: "Whether to return adjusted data. Only applicable to intraday\ + \ endpoints.\n" + default: false + order: 4 + outputsize: + title: "Output Size" + type: "string" + description: "Whether to return full or compact data (the last 100 data\ + \ points).\n" + enum: + - "compact" + - "full" + default: "compact" + order: 5 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-alloydb:1.0.15" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres"