Skip to content

Commit

Permalink
feat(targets): Allow users to disable schema validation in targets
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Jan 31, 2024
1 parent 32f3dec commit f1189ca
Show file tree
Hide file tree
Showing 4 changed files with 24 additions and 4 deletions.
11 changes: 11 additions & 0 deletions singer_sdk/helpers/capabilities.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,14 @@
default=False,
),
).to_dict()
TARGET_VALIDATE_SCHEMA_CONFIG = PropertiesList(
Property(
"validate_records",
BooleanType(),
description="Whether to validate the schema of the incoming streams.",
default=True,
),
).to_dict()


class TargetLoadMethods(str, Enum):
Expand Down Expand Up @@ -355,3 +363,6 @@ class TargetCapabilities(CapabilitiesEnum):

#: Allow setting the target schema.
TARGET_SCHEMA = "target-schema"

#: Validate the schema of the incoming streams.
VALIDATE_SCHEMA = "validate-schema"
13 changes: 10 additions & 3 deletions singer_sdk/sinks/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import json
import time
import typing as t
from functools import cached_property
from gzip import GzipFile
from gzip import open as gzip_open
from types import MappingProxyType
Expand Down Expand Up @@ -131,9 +132,6 @@ class Sink(metaclass=abc.ABCMeta):

MAX_SIZE_DEFAULT = 10000

validate_schema = True
"""Enable JSON schema record validation."""

validate_field_string_format = False
"""Enable JSON schema format validation, for example `date-time` string fields."""

Expand Down Expand Up @@ -186,6 +184,15 @@ def __init__(

self._validator: BaseJSONSchemaValidator | None = self.get_validator()

@cached_property
def validate_schema(self) -> bool:
"""Enable JSON schema record validation.
Returns:
True if JSON schema validation is enabled.
"""
return self.config.get("validate_records", True)

def get_validator(self) -> BaseJSONSchemaValidator | None:
"""Get a record validator for this sink.
Expand Down
2 changes: 2 additions & 0 deletions singer_sdk/target_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
TARGET_HARD_DELETE_CONFIG,
TARGET_LOAD_METHOD_CONFIG,
TARGET_SCHEMA_CONFIG,
TARGET_VALIDATE_SCHEMA_CONFIG,
CapabilitiesEnum,
PluginCapabilities,
TargetCapabilities,
Expand Down Expand Up @@ -608,6 +609,7 @@ def _merge_missing(source_jsonschema: dict, target_jsonschema: dict) -> None:

_merge_missing(ADD_RECORD_METADATA_CONFIG, config_jsonschema)
_merge_missing(TARGET_LOAD_METHOD_CONFIG, config_jsonschema)
_merge_missing(TARGET_VALIDATE_SCHEMA_CONFIG, config_jsonschema)

capabilities = cls.capabilities

Expand Down
2 changes: 1 addition & 1 deletion tests/core/targets/test_target_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,5 +46,5 @@ class MyTarget(SQLTargetMock, capabilities=capabilities):
pass

about = MyTarget._get_about_info()
default_settings = {"add_record_metadata", "load_method"}
default_settings = {"add_record_metadata", "load_method", "validate_schema"}
assert set(about.settings["properties"]) == expected_settings | default_settings

0 comments on commit f1189ca

Please sign in to comment.