Skip to content

Commit

Permalink
feat: Support validating configuration for any tap with a dynamic cat…
Browse files Browse the repository at this point in the history
…alog (#937)

* refactor: Use inheritance for plugins' CLI

* Make mypy happy

* feat: Support validating configuration for any tap with a dynamic catalog

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Update docs

* Delete leftover file

* Add # pragma: no cover

* Document class attributes

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
edgarrmondragon and pre-commit-ci[bot] authored Jan 19, 2024
1 parent 596323b commit ecdfa0e
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@

.. autoclass:: APIAuthenticatorBase
:members:
:special-members: __init__, __call__
:special-members: __init__, __call__
39 changes: 31 additions & 8 deletions singer_sdk/tap_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,11 @@

from singer_sdk._singerlib import Catalog, StateMessage
from singer_sdk.configuration._dict_config import merge_missing_config_jsonschema
from singer_sdk.exceptions import AbortedSyncFailedException, AbortedSyncPausedException
from singer_sdk.exceptions import (
AbortedSyncFailedException,
AbortedSyncPausedException,
ConfigValidationError,
)
from singer_sdk.helpers import _state
from singer_sdk.helpers._classproperty import classproperty
from singer_sdk.helpers._compat import final
Expand Down Expand Up @@ -53,6 +57,10 @@ class Tap(PluginBase, SingerWriter, metaclass=abc.ABCMeta):
plugins.
"""

dynamic_catalog: bool = False
"""Whether the tap's catalog is dynamic. Set to True if the catalog is
generated dynamically (e.g. by querying a database's system tables)."""

# Constructor

def __init__(
Expand Down Expand Up @@ -520,12 +528,17 @@ def cb_discover(

config_args = ctx.params.get("config", ())
config_files, parse_env_config = cls.config_from_cli_args(*config_args)
tap = cls(
config=config_files, # type: ignore[arg-type]
parse_env_config=parse_env_config,
validate_config=False,
setup_mapper=False,
)
try:
tap = cls(
config=config_files, # type: ignore[arg-type]
parse_env_config=parse_env_config,
validate_config=cls.dynamic_catalog,
setup_mapper=False,
)
except ConfigValidationError as exc: # pragma: no cover
for error in exc.errors:
cls.logger.error("Config validation error: %s", error)
ctx.exit(1)
tap.run_discovery()
ctx.exit()

Expand Down Expand Up @@ -611,8 +624,18 @@ def get_singer_command(cls: type[Tap]) -> click.Command:
class SQLTap(Tap):
"""A specialized Tap for extracting from SQL streams."""

# Stream class used to initialize new SQL streams from their catalog declarations.
default_stream_class: type[SQLStream]
"""
The default stream class used to initialize new SQL streams from their catalog
entries.
"""

dynamic_catalog: bool = True
"""
Whether the tap's catalog is dynamic, enabling configuration validation in
discovery mode. Set to True if the catalog is generated dynamically (e.g. by
querying a database's system tables).
"""

_tap_connector: SQLConnector | None = None

Expand Down

0 comments on commit ecdfa0e

Please sign in to comment.