Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Connector acceptance test: Fix discovered catalog caching for different configs #22301

Merged
merged 9 commits into from
Feb 6, 2023
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Changelog

## 0.5.1
Fix discovered catalog caching for different configs. [#22301](https://github.com/airbytehq/airbyte/pull/22301)

## 0.5.0
Re-release of 0.3.0 [#21451](https://github.com/airbytehq/airbyte/pull/21451)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
COPY connector_acceptance_test ./connector_acceptance_test
RUN pip install .

LABEL io.airbyte.version=0.5.0
LABEL io.airbyte.version=0.5.1
LABEL io.airbyte.name=airbyte/connector-acceptance-test

ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx"]
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
filter_output,
load_config,
load_yaml_or_json_path,
make_hashable,
)
from docker import errors

Expand Down Expand Up @@ -259,7 +260,10 @@ def discovered_catalog_fixture(
connector_config, docker_runner: ConnectorRunner, cached_schemas, cache_discovered_catalog: bool
) -> MutableMapping[str, AirbyteStream]:
"""JSON schemas for each stream"""
if not cached_schemas or not cache_discovered_catalog:
cached_schemas = cached_schemas.setdefault(make_hashable(connector_config), {})
if not cache_discovered_catalog:
cached_schemas.clear()
if not cached_schemas:
output = docker_runner.call_discover(config=connector_config)
catalogs = [message.catalog for message in output if message.type == Type.CATALOG]
for stream in catalogs[-1].streams:
Expand All @@ -269,14 +273,17 @@ def discovered_catalog_fixture(

@pytest.fixture(name="previous_discovered_catalog")
def previous_discovered_catalog_fixture(
connector_config, previous_connector_docker_runner: ConnectorRunner, previous_cached_schemas
connector_config, previous_connector_docker_runner: ConnectorRunner, previous_cached_schemas, cache_discovered_catalog: bool
) -> MutableMapping[str, AirbyteStream]:
"""JSON schemas for each stream"""
if previous_connector_docker_runner is None:
logging.warning(
"\n We could not retrieve the previous discovered catalog as a connector runner for the previous connector version could not be instantiated."
)
return None
previous_cached_schemas = previous_cached_schemas.setdefault(make_hashable(connector_config), {})
if not cache_discovered_catalog:
previous_cached_schemas.clear()
if not previous_cached_schemas:
output = previous_connector_docker_runner.call_discover(config=connector_config)
catalogs = [message.catalog for message in output if message.type == Type.CATALOG]
Expand Down