Skip to content

Commit

Permalink
fix spec command
Browse files Browse the repository at this point in the history
  • Loading branch information
brianjlai committed Oct 24, 2024
1 parent 96cf12b commit 2ca0f74
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,11 @@ def __init__(
self._concurrent_streams: List[AbstractStream]
self._synchronous_streams: List[Stream]

self._concurrent_streams, self._synchronous_streams = self._group_streams(config=config or {})
if config:
self._concurrent_streams, self._synchronous_streams = self._group_streams(config=config or {})
else:
self._concurrent_streams = []
self._synchronous_streams = []

concurrency_level_from_manifest = self._source_config.get("concurrency_level")
if concurrency_level_from_manifest:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@


class ModelToComponentFactory:

EPOCH_DATETIME_FORMAT = "%s"

def __init__(
self,
limit_pages_fetched_per_slice: Optional[int] = None,
Expand Down Expand Up @@ -521,15 +524,15 @@ def create_concurrent_cursor_from_datetime_based_cursor(
lookback_window = parse_duration(evaluated_lookback_window)

connector_state_converter: DateTimeStreamStateConverter
if datetime_format != "%s":
if datetime_format == self.EPOCH_DATETIME_FORMAT:
connector_state_converter = EpochValueConcurrentStreamStateConverter(is_sequential_state=True)
else:
connector_state_converter = CustomOutputFormatConcurrentStreamStateConverter(
datetime_format=datetime_format,
is_sequential_state=True,
cursor_granularity=cursor_granularity,
# type: ignore # Having issues w/ inspection for GapType and CursorValueType as shown in existing tests. Confirmed functionality is working in practice
)
else:
connector_state_converter = EpochValueConcurrentStreamStateConverter(is_sequential_state=True)

start_date_runtime_value: Union[InterpolatedString, str, MinMaxDatetime]
if isinstance(datetime_based_cursor_model.start_datetime, MinMaxDatetimeModel):
Expand Down
6 changes: 3 additions & 3 deletions airbyte-integrations/connectors/source-chargebee/poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
Expand Up @@ -474,5 +474,5 @@ check:
# customers can specify a higher concurrency level as needed up to the theoretical max rate limt.
concurrency_level:
type: ConcurrencyLevel
default_concurrency: "{{ config.num_workers or 3 }}"
default_concurrency: "{{ config.get('num_workers', 3) }}"
max_concurrency: 50

0 comments on commit 2ca0f74

Please sign in to comment.