-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[airbyte-cdk] Decouple request_options_provider from datetime_based_c…
…ursor + concurrent_cursor features for low-code (#45413)
- Loading branch information
Showing
12 changed files
with
1,030 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
...sources/declarative/requesters/request_options/datetime_based_request_options_provider.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,78 @@ | ||
# | ||
# Copyright (c) 2024 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
from dataclasses import InitVar, dataclass | ||
from typing import Any, Mapping, MutableMapping, Optional, Union | ||
|
||
from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString | ||
from airbyte_cdk.sources.declarative.requesters.request_option import RequestOption, RequestOptionType | ||
from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider | ||
from airbyte_cdk.sources.types import Config, StreamSlice, StreamState | ||
|
||
|
||
@dataclass | ||
class DatetimeBasedRequestOptionsProvider(RequestOptionsProvider): | ||
""" | ||
Request options provider that extracts fields from the stream_slice and injects them into the respective location in the | ||
outbound request being made | ||
""" | ||
|
||
config: Config | ||
parameters: InitVar[Mapping[str, Any]] | ||
start_time_option: Optional[RequestOption] = None | ||
end_time_option: Optional[RequestOption] = None | ||
partition_field_start: Optional[str] = None | ||
partition_field_end: Optional[str] = None | ||
|
||
def __post_init__(self, parameters: Mapping[str, Any]) -> None: | ||
self._partition_field_start = InterpolatedString.create(self.partition_field_start or "start_time", parameters=parameters) | ||
self._partition_field_end = InterpolatedString.create(self.partition_field_end or "end_time", parameters=parameters) | ||
|
||
def get_request_params( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Mapping[str, Any]: | ||
return self._get_request_options(RequestOptionType.request_parameter, stream_slice) | ||
|
||
def get_request_headers( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Mapping[str, Any]: | ||
return self._get_request_options(RequestOptionType.header, stream_slice) | ||
|
||
def get_request_body_data( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Union[Mapping[str, Any], str]: | ||
return self._get_request_options(RequestOptionType.body_data, stream_slice) | ||
|
||
def get_request_body_json( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Mapping[str, Any]: | ||
return self._get_request_options(RequestOptionType.body_json, stream_slice) | ||
|
||
def _get_request_options(self, option_type: RequestOptionType, stream_slice: Optional[StreamSlice]) -> Mapping[str, Any]: | ||
options: MutableMapping[str, Any] = {} | ||
if not stream_slice: | ||
return options | ||
if self.start_time_option and self.start_time_option.inject_into == option_type: | ||
options[self.start_time_option.field_name.eval(config=self.config)] = stream_slice.get( # type: ignore # field_name is always casted to an interpolated string | ||
self._partition_field_start.eval(self.config) | ||
) | ||
if self.end_time_option and self.end_time_option.inject_into == option_type: | ||
options[self.end_time_option.field_name.eval(config=self.config)] = stream_slice.get(self._partition_field_end.eval(self.config)) # type: ignore # field_name is always casted to an interpolated string | ||
return options |
58 changes: 58 additions & 0 deletions
58
...te_cdk/sources/declarative/requesters/request_options/default_request_options_provider.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
# | ||
# Copyright (c) 2024 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
from dataclasses import InitVar, dataclass | ||
from typing import Any, Mapping, Optional, Union | ||
|
||
from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider | ||
from airbyte_cdk.sources.types import StreamSlice, StreamState | ||
|
||
|
||
@dataclass | ||
class DefaultRequestOptionsProvider(RequestOptionsProvider): | ||
""" | ||
Request options provider that extracts fields from the stream_slice and injects them into the respective location in the | ||
outbound request being made | ||
""" | ||
|
||
parameters: InitVar[Mapping[str, Any]] | ||
|
||
def __post_init__(self, parameters: Mapping[str, Any]) -> None: | ||
pass | ||
|
||
def get_request_params( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Mapping[str, Any]: | ||
return {} | ||
|
||
def get_request_headers( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Mapping[str, Any]: | ||
return {} | ||
|
||
def get_request_body_data( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Union[Mapping[str, Any], str]: | ||
return {} | ||
|
||
def get_request_body_json( | ||
self, | ||
*, | ||
stream_state: Optional[StreamState] = None, | ||
stream_slice: Optional[StreamSlice] = None, | ||
next_page_token: Optional[Mapping[str, Any]] = None, | ||
) -> Mapping[str, Any]: | ||
return {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.