Skip to content

Commit

Permalink
Source Marketo: set high test strictness level (#21790)
Browse files Browse the repository at this point in the history
* #19055 source marketo: set high test strictness level

* #19055 source marketo: increase incremental test timeout

* #19055 source Marketo: fix stream schemas

* #19055 source marketoL fix unit test

* Updated expected records

* Updated expected records

Co-authored-by: Serhii Lazebnyi <[email protected]>
  • Loading branch information
davydov-d and lazebnyi authored Jan 26, 2023
1 parent b0a81ba commit 34d43cc
Show file tree
Hide file tree
Showing 9 changed files with 313 additions and 165 deletions.
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-marketo/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.11
LABEL io.airbyte.version=1.0.0
LABEL io.airbyte.name=airbyte/source-marketo
Original file line number Diff line number Diff line change
@@ -1,29 +1,82 @@
# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
# for more information about how to configure these tests
connector_image: airbyte/source-marketo:dev
tests:
test_strictness_level: high
acceptance_tests:
spec:
- spec_path: "source_marketo/spec.json"
tests:
- spec_path: "source_marketo/spec.json"
connection:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
tests:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
discovery:
- config_path: "secrets/config.json"
tests:
- config_path: "secrets/config.json"
basic_read:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
empty_streams: ["lists", "campaigns"]
timeout_seconds: 3600
expect_records:
path: "integration_tests/expected_records.jsonl"
tests:
- config_path: "secrets/config.json"
expect_records:
path: "integration_tests/expected_records.jsonl"
empty_streams:
- name: "activities_update_opportunity"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_change_segment"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_document_interacted_with"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_call_webhook"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_push_leadto_marketo"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_change_revenue_stage"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_dialogue_goal_reached"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_removefrom_opportunity"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_received_forwardto_friend_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_open_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_share_content"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_sales_email_bounced"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_dialogue_engaged"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_dialogue_appointment_scheduled"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_click_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_sent_forwardto_friend_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_click_link"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_send_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_change_revenue_stage_manually"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_receive_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_replyto_sales_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_addto_opportunity"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
- name: "activities_open_email"
bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction"
# 52 streams, most of them use BULK API therefore it takes much time to run a sync
timeout_seconds: 9000
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/incremental_catalog.json"
future_state_path: "integration_tests/abnormal_state.json"
timeout_seconds: 3600
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/incremental_catalog.json"
future_state:
future_state_path: "integration_tests/abnormal_state.json"
timeout_seconds: 3600
full_refresh:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"
timeout_seconds: 3600
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/configured_catalog.json"

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -4,65 +4,84 @@
"stream": {
"name": "programs",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["updatedAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["updatedAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "campaigns",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["createdAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["createdAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "lists",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["createdAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["createdAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "leads",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"supported_sync_modes": ["full_refresh", "incremental"],
"default_cursor_field": ["updatedAt"],
"source_defined_primary_key": [["id"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["updatedAt"],
"primary_key": [["id"]]
},
{
"stream": {
"name": "activity_types",
"json_schema": {},
"supported_sync_modes": ["full_refresh"]
"supported_sync_modes": ["full_refresh"],
"source_defined_cursor": false,
"default_cursor_field": null,
"source_defined_primary_key": [["id"]]
},
"sync_mode": "full_refresh",
"destination_sync_mode": "append"
"destination_sync_mode": "append",
"cursor_field": null,
"primary_key": [["id"]]
},
{
"stream": {
"name": "activities_visit_webpage",
"json_schema": {},
"supported_sync_modes": ["incremental"],
"supported_sync_modes": ["full_refresh", "incremental"],
"source_defined_cursor": true,
"default_cursor_field": []
"default_cursor_field": ["activityDate"],
"source_defined_primary_key": [["marketoGUID"]]
},
"sync_mode": "incremental",
"destination_sync_mode": "append"
"sync_mode": "full_refresh",
"destination_sync_mode": "append",
"cursor_field": ["activityDate"],
"primary_key": [["marketoGUID"]]
}
]
}
Loading

0 comments on commit 34d43cc

Please sign in to comment.