Preserving headers in SaaSRequestParams during pagination

CHANGELOG.md

@@ -19,6 +19,10 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.7.0...main)
 
+### Fixed
+
+* HTTP headers are now preserved in requests generated from SaaS connector pagination [#1069](https://github.com/ethyca/fidesops/pull/1069)
+
 ## [1.7.0](https://github.com/ethyca/fidesops/compare/1.6.3...1.7.0)
 
 ### Added

src/fidesops/ops/service/pagination/pagination_strategy_cursor.py

@@ -47,6 +47,7 @@ def get_next_request(
 
         return SaaSRequestParams(
             method=request_params.method,
+            headers=request_params.headers,
             path=request_params.path,
             query_params=request_params.query_params,
             body=request_params.body,

src/fidesops/ops/service/pagination/pagination_strategy_link.py

@@ -63,6 +63,7 @@ def get_next_request(
         )
         return SaaSRequestParams(
             method=request_params.method,
+            headers=request_params.headers,
             path=updated_path,
             query_params=updated_query_params,
             body=request_params.body,

src/fidesops/ops/service/pagination/pagination_strategy_offset.py

@@ -63,6 +63,7 @@ def get_next_request(
         request_params.query_params[self.incremental_param] = param_value
         return SaaSRequestParams(
             method=request_params.method,
+            headers=request_params.headers,
             path=request_params.path,
             query_params=request_params.query_params,
             body=request_params.body,

tests/ops/service/pagination/test_pagination_strategy_cursor.py

@@ -75,3 +75,17 @@ def test_cursor_with_empty_list(response_with_empty_list):
         request_params, {}, response_with_empty_list, "conversations"
     )
     assert next_request is None
+
+
+def test_headers_present_in_paginated_request(response_with_body):
+    config = CursorPaginationConfiguration(cursor_param="after", field="id")
+    request_params: SaaSRequestParams = SaaSRequestParams(
+        method=HTTPMethod.GET,
+        headers={"X-Fides-Token": "token"},
+        path="/conversations",
+    )
+    paginator = CursorPaginationStrategy(config)
+    next_request: SaaSRequestParams = paginator.get_next_request(
+        request_params, {}, response_with_body, "conversations"
+    )
+    assert next_request.headers == request_params.headers

tests/ops/service/pagination/test_pagination_strategy_link.py

@@ -154,3 +154,19 @@ def test_config_mismatch():
         "The 'path' value must be specified when accessing the link from the body"
         in str(exc.value)
     )
+
+
+def test_headers_present_in_paginated_request(response_with_body_link):
+    config = LinkPaginationConfiguration(source="body", path="links.next")
+    request_params: SaaSRequestParams = SaaSRequestParams(
+        method=HTTPMethod.GET,
+        headers={"x-custom-header": "abc"},
+        path="/customers",
+        query_params={"page": "abc"},
+    )
+
+    paginator = LinkPaginationStrategy(config)
+    next_request: Optional[SaaSRequestParams] = paginator.get_next_request(
+        request_params, {}, response_with_body_link, "customers"
+    )
+    assert next_request.headers == request_params.headers

tests/ops/service/pagination/test_pagination_strategy_offset.py

@@ -176,3 +176,21 @@ def test_validate_request_missing_param():
             method="GET", path="/test", query_params=query_params, pagination=pagination
         )
     assert "Query param 'page' not found." in str(exc.value)
+
+
+def test_headers_present_in_paginated_request(response_with_body):
+    config = OffsetPaginationConfiguration(
+        incremental_param="page", increment_by=1, limit=10
+    )
+    request_params: SaaSRequestParams = SaaSRequestParams(
+        method=HTTPMethod.GET,
+        headers={"X-Fides-Token": "token"},
+        path="/conversations",
+        query_params={"page": 1},
+    )
+
+    paginator = OffsetPaginationStrategy(config)
+    next_request: Optional[SaaSRequestParams] = paginator.get_next_request(
+        request_params, {}, response_with_body, "conversations"
+    )
+    assert next_request.headers == request_params.headers