From 8c039f45dff1c8211a4e524db05d3a2b342d10d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez-Mondrag=C3=B3n?= Date: Mon, 5 Feb 2024 12:50:31 -0600 Subject: [PATCH] refactor: Experiment with default offset paginator --- tap_dbt/streams.py | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/tap_dbt/streams.py b/tap_dbt/streams.py index 6910b02..506c855 100644 --- a/tap_dbt/streams.py +++ b/tap_dbt/streams.py @@ -11,35 +11,9 @@ from tap_dbt.client import DBTStream -if t.TYPE_CHECKING: - import requests - SCHEMAS_DIR = Path(__file__).parent / Path("./schemas") -class DbtPaginator(BaseOffsetPaginator): - """dbt API paginator.""" - - def has_more(self, response: requests.Response) -> bool: - """Returns True until there are no more pages to retrieve. - - The API returns an 'extra' key with information about pagination: - "extra":{"filters":{"limit":100,"offset":2,"account_id":1},"order_by":"id","pagination":{"count":100,"total_count":209}}} - """ - data = response.json() - extra = data.get("extra", {}) - filters = extra.get("filters", {}) - pagination = extra.get("pagination", {}) - - offset = filters.get("offset", 0) - total_count = pagination.get("total_count") - count = pagination.get("count") - - # The pagination has more records when: - # total_count is still greater than count and offset combined - return count + offset < total_count - - class AccountBasedStream(DBTStream): """A stream that requires an account ID.""" @@ -59,7 +33,7 @@ def partitions(self) -> list[dict]: ) raise ValueError(errmsg) - def get_new_paginator(self) -> DbtPaginator: + def get_new_paginator(self) -> BaseOffsetPaginator: """Return a new paginator instance for this stream.""" page_size = self.config["page_size"] @@ -68,7 +42,7 @@ def get_new_paginator(self) -> DbtPaginator: page_size, ) - return DbtPaginator(start_value=0, page_size=page_size) + return BaseOffsetPaginator(start_value=0, page_size=page_size) def get_url_params( self,