Skip to content

Commit

Permalink
converting floats to decimals before schema validation
Browse files Browse the repository at this point in the history
  • Loading branch information
taylorbarstow committed Nov 11, 2023
1 parent 466d60a commit d55ffe2
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 1 deletion.
13 changes: 13 additions & 0 deletions singer_sdk/helpers/_typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import datetime
import logging
import typing as t
from decimal import Decimal
from enum import Enum
from functools import lru_cache

Expand Down Expand Up @@ -491,3 +492,15 @@ def _conform_primitive_property( # noqa: PLR0911
if is_boolean_type(property_schema):
return None if elem is None else elem != 0
return elem

def float_to_decimal(value):
"""
Walk the given data structure and turn all instances of float into double.
"""
if isinstance(value, float):
return Decimal(str(value))
if isinstance(value, list):
return [float_to_decimal(child) for child in value]
if isinstance(value, dict):
return {k: float_to_decimal(v) for k, v in value.items()}
return value
3 changes: 2 additions & 1 deletion singer_sdk/sinks/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
DatetimeErrorTreatmentEnum,
get_datelike_property_type,
handle_invalid_timestamp_in_record,
float_to_decimal,
)

if t.TYPE_CHECKING:
Expand Down Expand Up @@ -321,7 +322,7 @@ def _validate_and_parse(self, record: dict) -> dict:
Returns:
TODO
"""
self._validator.validate(record)
self._validator.validate(float_to_decimal(record))
self._parse_timestamps_in_record(
record=record,
schema=self.schema,
Expand Down

0 comments on commit d55ffe2

Please sign in to comment.