Skip to content

Commit

Permalink
docs: Document supported package extras
Browse files Browse the repository at this point in the history
  • Loading branch information
edgarrmondragon committed Jan 31, 2024
1 parent 2fbe530 commit 49a0f77
Show file tree
Hide file tree
Showing 7 changed files with 31 additions and 43 deletions.
2 changes: 1 addition & 1 deletion docs/batch.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ AWS S3

### `encoding`

The `encoding` field is used to specify the format and compression of the batch files. Currently only `jsonl` and `gzip` are supported, respectively.
The `encoding` field is used to specify the format and compression of the batch files. Currently `jsonl`, `gzip` and `parquet` are supported.

### `manifest`

Expand Down
8 changes: 8 additions & 0 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
"sphinx.ext.napoleon",
"sphinx.ext.autosectionlabel",
"sphinx.ext.autosummary",
"sphinx.ext.intersphinx",
"sphinx_copybutton",
"myst_parser",
"sphinx_reredirects",
Expand Down Expand Up @@ -127,3 +128,10 @@
redirects = {
"porting.html": "guides/porting.html",
}

# -- Options for intersphinx -------------------------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
intersphinx_mapping = {
"requests": ("https://requests.readthedocs.io/en/latest/", None),
"python": ("https://docs.python.org/3/", None),
}
8 changes: 8 additions & 0 deletions docs/dev_guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,14 @@ Some APIs instead return the records as values inside an object where each key i
]
```

## Extra features

The following [extra features](https://packaging.python.org/en/latest/specifications/dependency-specifiers/#extras) are available for the Singer SDK:

- `s3` - Enables AWS S3 as a [BATCH storage](batch.md#the-batch-message).
- `parquet` - Enables as [BATCH encoding](batch.md#encoding).
- `testing` - Pytest dependencies required to use the [Tap & Target Testing Framework](testing.md).

## Resources

### Detailed Class Reference
Expand Down
10 changes: 2 additions & 8 deletions singer_sdk/authenticators.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,13 +131,10 @@ def authenticate_request(
"""Authenticate a request.
Args:
request: A `request object`_.
request: A :class:`requests.PreparedRequest` object.
Returns:
The authenticated request object.
.. _request object:
https://requests.readthedocs.io/en/latest/api/#requests.PreparedRequest
"""
request.headers.update(self.auth_headers)

Expand All @@ -154,13 +151,10 @@ def __call__(self, r: requests.PreparedRequest) -> requests.PreparedRequest:
and returns the result.
Args:
r: A `request object`_.
r: A :class:`requests.PreparedRequest` object.
Returns:
The authenticated request object.
.. _request object:
https://requests.readthedocs.io/en/latest/api/#requests.PreparedRequest
"""
return self.authenticate_request(r)

Expand Down
5 changes: 1 addition & 4 deletions singer_sdk/pagination.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ class BaseHATEOASPaginator(
like "https://api.com/link/to/next-item".
The :attr:`~singer_sdk.pagination.BaseAPIPaginator.current_value` attribute of
this paginator is a `urllib.parse.ParseResult`_ object. This object
this paginator is a :class:`urllib.parse.ParseResult` object. This object
contains the following attributes:
- scheme
Expand All @@ -208,9 +208,6 @@ def get_url_params(self, next_page_token) -> dict:
if next_page_token:
return dict(parse_qsl(next_page_token.query))
return {}
.. _`urllib.parse.ParseResult`:
https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlparse
"""

def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
Expand Down
4 changes: 1 addition & 3 deletions singer_sdk/plugin_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -403,9 +403,7 @@ def print_version(
Args:
print_fn: A function to use to display the plugin version.
Defaults to `print`_.
.. _print: https://docs.python.org/3/library/functions.html#print
Defaults to :py:func:`print`.
"""
print_fn(f"{cls.name} v{cls.plugin_version}, Meltano SDK v{cls.sdk_version}")

Expand Down
37 changes: 10 additions & 27 deletions singer_sdk/streams/rest.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,10 +137,7 @@ def requests_session(self) -> requests.Session:
"""Get requests session.
Returns:
The `requests.Session`_ object for HTTP requests.
.. _requests.Session:
https://requests.readthedocs.io/en/latest/api.html#requests.Session
The :class:`requests.Session` object for HTTP requests.
"""
if not self._requests_session:
self._requests_session = requests.Session()
Expand Down Expand Up @@ -168,14 +165,11 @@ def validate_response(self, response: requests.Response) -> None:
.. image:: ../images/200.png
Args:
response: A `requests.Response`_ object.
response: A :class:`requests.Response` object.
Raises:
FatalAPIError: If the request is not retriable.
RetriableAPIError: If the request is retriable.
.. _requests.Response:
https://requests.readthedocs.io/en/latest/api.html#requests.Response
"""
if (
response.status_code in self.extra_retry_statuses
Expand All @@ -198,7 +192,7 @@ def response_error_message(self, response: requests.Response) -> str:
WARNING - Override this method when the URL path may contain secrets or PII
Args:
response: A `requests.Response`_ object.
response: A :class:`requests.Response` object.
Returns:
str: The error message
Expand Down Expand Up @@ -286,7 +280,7 @@ def get_url_params(
If your source needs special handling and, for example, parentheses should not
be encoded, you can return a string constructed with
`urllib.parse.urlencode`_:
:py:func:`urllib.parse.urlencode`:
.. code-block:: python
Expand All @@ -306,9 +300,6 @@ def get_url_params(self, context, next_page_token):
Returns:
Dictionary or encoded string with URL query parameters to use in the
request.
.. _urllib.parse.urlencode:
https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode
"""
return {}

Expand All @@ -322,16 +313,11 @@ def build_prepared_request(
Uses the authenticator instance to mutate the request with authentication.
Args:
*args: Arguments to pass to `requests.Request`_.
**kwargs: Keyword arguments to pass to `requests.Request`_.
*args: Arguments to pass to :class:`requests.Request`.
**kwargs: Keyword arguments to pass to :class:`requests.Request`.
Returns:
A `requests.PreparedRequest`_ object.
.. _requests.PreparedRequest:
https://requests.readthedocs.io/en/latest/api.html#requests.PreparedRequest
.. _requests.Request:
https://requests.readthedocs.io/en/latest/api.html#requests.Request
A :class:`requests.PreparedRequest` object.
"""
request = requests.Request(*args, **kwargs)
self.requests_session.auth = self.authenticator
Expand Down Expand Up @@ -460,7 +446,7 @@ def update_sync_costs(
Args:
request: the Request object that was just called.
response: the `requests.Response` object
response: the :class:`requests.Response` object
context: the context passed to the call
Returns:
Expand Down Expand Up @@ -498,7 +484,7 @@ def calculate_sync_cost(
Args:
request: the API Request object that was just called.
response: the `requests.Response` object
response: the :class:`requests.Response` object
context: the context passed to the call
Returns:
Expand Down Expand Up @@ -596,13 +582,10 @@ def parse_response(self, response: requests.Response) -> t.Iterable[dict]:
"""Parse the response and return an iterator of result records.
Args:
response: A raw `requests.Response`_ object.
response: A raw :class:`requests.Response`
Yields:
One item for every item found in the response.
.. _requests.Response:
https://requests.readthedocs.io/en/latest/api.html#requests.Response
"""
yield from extract_jsonpath(self.records_jsonpath, input=response.json())

Expand Down

0 comments on commit 49a0f77

Please sign in to comment.