Merge branch 'refactor/cli-oop' into fix/configurable-dynamic-catalog

.github/workflows/constraints.txt

@@ -1,5 +1,5 @@
 pip==22.2.2
-poetry==1.1.15
-virtualenv==20.16.3
+poetry==1.2.0
+virtualenv==20.16.4
 nox==2022.8.7
 nox-poetry==1.0.1

.github/workflows/pr-preview-links.yml

@@ -0,0 +1,17 @@
+name: Read the Docs Pull Request Preview
+
+on:
+  pull_request_target:
+    types:
+    - opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  pr-preview-links:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: readthedocs/actions/preview@v1
+      with:
+        project-slug: "meltano-sdk"

docs/classes/singer_sdk.exceptions.RecordsWithoutSchemaException.rst

@@ -0,0 +1,7 @@
+singer_sdk.exceptions.RecordsWithoutSchemaException
+==================================================
+
+.. currentmodule:: singer_sdk.exceptions
+
+.. autoclass:: RecordsWithoutSchemaException
+    :members:

docs/classes/singer_sdk.pagination.BaseAPIPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.BaseAPIPaginator
+======================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: BaseAPIPaginator
+    :members:

docs/classes/singer_sdk.pagination.BaseHATEOASPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.BaseHATEOASPaginator
+==========================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: BaseHATEOASPaginator
+    :members:

docs/classes/singer_sdk.pagination.BaseOffsetPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.BaseOffsetPaginator
+=========================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: BaseOffsetPaginator
+    :members:

docs/classes/singer_sdk.pagination.BasePageNumberPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.BasePageNumberPaginator
+=============================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: BasePageNumberPaginator
+    :members:

docs/classes/singer_sdk.pagination.HeaderLinkPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.HeaderLinkPaginator
+=========================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: HeaderLinkPaginator
+    :members:

docs/classes/singer_sdk.pagination.JSONPathPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.JSONPathPaginator
+=======================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: JSONPathPaginator
+    :members:

docs/classes/singer_sdk.pagination.LegacyPaginatedStreamProtocol.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.LegacyPaginatedStreamProtocol
+===================================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: LegacyPaginatedStreamProtocol
+    :members:

docs/classes/singer_sdk.pagination.LegacyStreamPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.LegacyStreamPaginator
+===========================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: LegacyStreamPaginator
+    :members:

docs/classes/singer_sdk.pagination.SimpleHeaderPaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.SimpleHeaderPaginator
+===========================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: SimpleHeaderPaginator
+    :members:

docs/classes/singer_sdk.pagination.SinglePagePaginator.rst

@@ -0,0 +1,7 @@
+singer_sdk.pagination.SinglePagePaginator
+=========================================
+
+.. currentmodule:: singer_sdk.pagination
+
+.. autoclass:: SinglePagePaginator
+    :members:

docs/porting.md

@@ -103,10 +103,82 @@ _Important: If you've gotten this far, this is a good time to commit your code b
 
 Pagination is generally unique for almost every API. There's no single method that solves for very different API's approach to pagination.
 
-Most likely you will use `get_next_page_token` to parse and return whatever the "next page" token is for your source, and you'll use `get_url_params` to define how to pass the "next page" token back to the API when asking for subsequent pages.
+Most likely you will use [get_new_paginator](singer_sdk.RESTStream.get_new_paginator) to instantiate a [pagination class](./classes/singer_sdk.pagination.BaseAPIPaginator) for your source, and you'll use `get_url_params` to define how to pass the "next page" token back to the API when asking for subsequent pages.
 
 When you think you have it right, run `poetry run tap-mysource` again, and debug until you are confident the result is including multiple pages back from the API.
 
+You can also add unit tests for your pagination implementation for additional confidence:
+
+```python
+from singer_sdk.pagination import BaseHATEOASPaginator, first
+
+
+class CustomHATEOASPaginator(BaseHATEOASPaginator):
+   """Paginator for HATEOAS APIs - or "Hypermedia as the Engine of Application State".
+
+   This paginator expects responses to have a key "next" with a value
+   like "https://api.com/link/to/next-item".
+   """"
+
+   def get_next_url(self, response: Response) -> str | None:
+      """Get a parsed HATEOAS link for the next, if the response has one."""
+
+      try:
+            return first(
+               extract_jsonpath("$.links[?(@.rel=='next')].href", response.json())
+            )
+      except StopIteration:
+            return None
+
+
+def test_paginator_custom_hateoas():
+   """Validate paginator that my custom paginator."""
+
+   resource_path = "/path/to/resource"
+   response = Response()
+   paginator = CustomHATEOASPaginator()
+   assert not paginator.finished
+   assert paginator.current_value is None
+   assert paginator.count == 0
+
+   response._content = json.dumps(
+      {
+         "links": [
+               {
+                  "rel": "next",
+                  "href": f"{resource_path}?page=2&limit=100",
+               }
+         ]
+      }
+   ).encode()
+   paginator.advance(response)
+   assert not paginator.finished
+   assert paginator.current_value.path == resource_path
+   assert paginator.current_value.query == "page=2&limit=100"
+   assert paginator.count == 1
+
+   response._content = json.dumps(
+      {
+         "links": [
+               {
+                  "rel": "next",
+                  "href": f"{resource_path}?page=3&limit=100",
+               }
+         ]
+      }
+   ).encode()
+   paginator.advance(response)
+   assert not paginator.finished
+   assert paginator.current_value.path == resource_path
+   assert paginator.current_value.query == "page=3&limit=100"
+   assert paginator.count == 2
+
+   response._content = json.dumps({"links": []}).encode()
+   paginator.advance(response)
+   assert paginator.finished
+   assert paginator.count == 3
+```
+
 Note: Depending on how well the API is designed, this could take 5 minutes or multiple hours. If you need help, sometimes [PostMan](https://postman.com) or [Thunder Client](https://marketplace.visualstudio.com/items?itemName=rangav.vscode-thunder-client) can be helpful in debugging the APIs specific quirks.
 
 ## Run pytest

docs/reference.rst

@@ -76,7 +76,7 @@ Exception Types
     exceptions.InvalidStreamSortException
     exceptions.MapExpressionError
     exceptions.MaxRecordsLimitException
-    exceptions.RecordsWitoutSchemaException
+    exceptions.RecordsWithoutSchemaException
     exceptions.RetriableAPIError
     exceptions.StreamMapConfigError
     exceptions.TapStreamConnectionFailure
@@ -89,3 +89,22 @@ JSON Schema builder classes
     :template: module.rst
 
     typing
+
+
+Pagination
+----------
+
+.. autosummary::
+    :toctree: classes
+    :template: class.rst
+
+    pagination.BaseAPIPaginator
+    pagination.SinglePagePaginator
+    pagination.BaseHATEOASPaginator
+    pagination.HeaderLinkPaginator
+    pagination.JSONPathPaginator
+    pagination.SimpleHeaderPaginator
+    pagination.BasePageNumberPaginator
+    pagination.BaseOffsetPaginator
+    pagination.LegacyPaginatedStreamProtocol
+    pagination.LegacyStreamPaginator