diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index a462bf91612c..5894fff70338 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,9 +1,11 @@ # Changelog +## 0.2.5 +Unit test `test_state_with_abnormally_large_values` to check state emission testing is working. + ## 0.2.4 Make incremental tests compatible with per stream states.[#16686](https://github.com/airbytehq/airbyte/pull/16686/) - ## 0.2.3 Backward compatibility tests: improve `check_if_type_of_type_field_changed` to make it less radical when validating specs and allow `'str' -> ['str', '']` type changes.[#16429](https://github.com/airbytehq/airbyte/pull/16429/) diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 532ccd74f05d..049e3ebe2e57 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py index 3bf2feeda18c..fe061812b409 100644 --- a/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/source-acceptance-test/unit_tests/test_incremental.py @@ -613,3 +613,71 @@ def test_config_skip_test(): # This is guaranteed to fail when the test gets executed docker_runner_mock.call_read.assert_not_called() + + +@pytest.mark.parametrize( + "read_output, expectation", + [ + pytest.param([], pytest.raises(AssertionError), id="Error because incremental stream should always emit state messages"), + pytest.param( + [ + AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={"date": "2022-10-04"}, emitted_at=111) + ), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="test_stream"), + stream_state=AirbyteStateBlob.parse_obj({"date": "2022-10-04"}), + ), + data={"date": "2022-10-04"}, + ), + ), + ], + pytest.raises(AssertionError), + id="Error because incremental sync with abnormally large state value should not produce record.", + ), + pytest.param( + [ + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="test_stream"), + stream_state=AirbyteStateBlob.parse_obj({"date": "2022-10-04"}), + ), + data={"date": "2022-10-04"}, + ), + ) + ], + does_not_raise(), + ), + ], +) +def test_state_with_abnormally_large_values(mocker, read_output, expectation): + docker_runner_mock = mocker.MagicMock() + docker_runner_mock.call_read_with_state.return_value = read_output + t = _TestIncremental() + with expectation: + t.test_state_with_abnormally_large_values( + connector_config=mocker.MagicMock(), + configured_catalog=ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="test_stream", + json_schema={"type": "object", "properties": {"date": {"type": "date"}}}, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + ), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + cursor_field=["date"], + ) + ] + ), + future_state=mocker.MagicMock(), + docker_runner=docker_runner_mock, + ) diff --git a/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml index 3f9d1efe2ee2..df5768d18531 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml @@ -18,7 +18,7 @@ tests: incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json index ac43461345a0..7dffb12b07f6 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json @@ -1,23 +1,79 @@ -{ - "customers": { - "date_modified": "2080-07-30T22:16:46" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2080-07-30T22:16:46" + }, + "stream_descriptor": { + "name": "customers" + } + } }, - "orders": { - "date_modified": "2080-07-30T22:16:46+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2080-07-30T22:16:46+00:00" + }, + "stream_descriptor": { + "name": "orders" + } + } }, - "transactions": { - "id": 5783048848838997 + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 5783048848838997 + }, + "stream_descriptor": { + "name": "transactions" + } + } }, - "pages": { - "id": 9000002039398998 + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 9000002039398998 + }, + "stream_descriptor": { + "name": "pages" + } + } }, - "products": { - "date_modified": "2080-01-10T00:17:08+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2080-01-10T00:17:08+00:00" + }, + "stream_descriptor": { + "name": "products" + } + } }, - "channels": { - "date_modified": "2080-01-10T00:17:08+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2080-01-10T00:17:08+00:00" + }, + "stream_descriptor": { + "name": "channels" + } + } }, - "store": { - "store_id": 99199991999999199 + { + "type": "STREAM", + "stream": { + "stream_state": { + "store_id": 99199991999999200 + }, + "stream_descriptor": { + "name": "store" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json index 908c17c29d8d..b9495cc6af9f 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/sample_state.json @@ -1,23 +1,79 @@ -{ - "customers": { - "date_modified": "2021-08-11T15:18:55Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2021-08-11T15:18:55Z" + }, + "stream_descriptor": { + "name": "customers" + } + } }, - "orders": { - "date_modified": "2021-08-16T21:09:10+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2021-08-16T21:09:10+00:00" + }, + "stream_descriptor": { + "name": "orders" + } + } }, - "transactions": { - "id": 0 + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 0 + }, + "stream_descriptor": { + "name": "transactions" + } + } }, - "pages": { - "id": 4 + { + "type": "STREAM", + "stream": { + "stream_state": { + "id": 4 + }, + "stream_descriptor": { + "name": "pages" + } + } }, - "products": { - "date_modified": "2022-01-10T00:17:08+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2022-01-10T00:17:08+00:00" + }, + "stream_descriptor": { + "name": "products" + } + } }, - "channels": { - "date_modified": "2022-01-10T00:17:08+00:00" + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_modified": "2022-01-10T00:17:08+00:00" + }, + "stream_descriptor": { + "name": "channels" + } + } }, - "store": { - "store_id": 0 + { + "type": "STREAM", + "stream": { + "stream_state": { + "store_id": 0 + }, + "stream_descriptor": { + "name": "store" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml index 3c59848ec0aa..78a5be12a22c 100644 --- a/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paystack/acceptance-test-config.yml @@ -12,10 +12,11 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ['disputes', 'transfers', 'settlements', 'invoices'] + empty_streams: ["disputes", "transfers", "settlements", "invoices"] incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..62a1e078bdfc --- /dev/null +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/abnormal_state.json @@ -0,0 +1,90 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "customers" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "disputes" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "created_at": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "invoices" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "refunds" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "settlements" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "subscriptions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "transactions" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdAt": "2099-08-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "transfers" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json index 2b7a6496036c..8eebf5980873 100644 --- a/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paystack/integration_tests/sample_state.json @@ -1,10 +1,58 @@ -{ - "customers": { "createdAt": "2019-08-01T00:00:00Z" }, - "disputes": { "createdAt": "2019-08-01T00:00:00Z" }, - "invoices": { "created_at": "2019-08-01T00:00:00Z" }, - "refunds": { "createdAt": "2019-08-01T00:00:00Z" }, - "settlements": { "createdAt": "2019-08-01T00:00:00Z" }, - "subscriptions": { "createdAt": "2019-08-01T00:00:00Z" }, - "transactions": { "createdAt": "2019-08-01T00:00:00Z" }, - "transfers": { "createdAt": "2019-08-01T00:00:00Z" } -} +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "customers" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "disputes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "invoices" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "refunds" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "settlements" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "subscriptions" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "transactions" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "createdAt": "2019-08-01T00:00:00Z" }, + "stream_descriptor": { "name": "transfers" } + } + } +] diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml index fd2f549bb517..0feabb8df976 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-config.yml @@ -15,6 +15,7 @@ tests: incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-config.yml b/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-config.yml index c6c27a98f3f5..6cc635226775 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-config.yml @@ -15,15 +15,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.txt" - # extra_fields: no - # exact_order: no - # extra_records: yes incremental: # TODO if your connector does not implement incremental sync, remove this block - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - # full_refresh: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-talkdesk-explore/integration_tests/sample_state.json index 3587e579822d..2489355851a2 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-talkdesk-explore/integration_tests/sample_state.json @@ -1,5 +1,57 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "end_at": "2099-01-01T00:00:00" + }, + "stream_descriptor": { + "name": "calls" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "status_end_at": "2099-01-01T00:00:00" + }, + "stream_descriptor": { + "name": "user_status" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "studio_flow_executions_aggregated.flow_execution_finished_time": "2099-01-01T00:00:00" + }, + "stream_descriptor": { + "name": "studio_flow_execution" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "finished_at": "2099-01-01T00:00:00" + }, + "stream_descriptor": { + "name": "contacts" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "ring_finished_at_time": "2099-01-01T00:00:00" + }, + "stream_descriptor": { + "name": "ring_attempts" + } + } } -} +]