Skip to content

Commit

Permalink
Fix pagination issue in JSONResponseCursorPaginator with empty stri…
Browse files Browse the repository at this point in the history
…ng cursor value

Adjusts the `update_state()` method to set `_next_reference` to None
when the cursor value extracted from the JSON response is an empty
string, preventing unintended pagination requests.

Fixed #2012
  • Loading branch information
kang8 committed Nov 2, 2024
1 parent 4312332 commit 3c7919e
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 1 deletion.
2 changes: 1 addition & 1 deletion dlt/sources/helpers/rest_client/paginators.py
Original file line number Diff line number Diff line change
Expand Up @@ -651,7 +651,7 @@ def __init__(
def update_state(self, response: Response, data: Optional[List[Any]] = None) -> None:
"""Extracts the cursor value from the JSON response."""
values = jsonpath.find_values(self.cursor_path, response.json())
self._next_reference = values[0] if values else None
self._next_reference = values[0] if values and values[0] else None

def update_request(self, request: Request) -> None:
"""Updates the request with the cursor query parameter."""
Expand Down
6 changes: 6 additions & 0 deletions tests/sources/helpers/rest_client/test_paginators.py
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,12 @@ def test_update_state(self):
assert paginator._next_reference == "cursor-2"
assert paginator.has_next_page is True

def test_update_state_when_cursor_path_is_empty_string(self):
paginator = JSONResponseCursorPaginator(cursor_path="next_cursor")
response = Mock(Response, json=lambda: {"next_cursor": "", "results": []})
paginator.update_state(response)
assert paginator.has_next_page is False

def test_update_request(self):
paginator = JSONResponseCursorPaginator(cursor_path="next_cursor")
paginator._next_reference = "cursor-2"
Expand Down

0 comments on commit 3c7919e

Please sign in to comment.