From 7977269e8d58672d3b5467e2582237347bb074a2 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Sat, 27 Jul 2024 18:44:01 -0700 Subject: [PATCH] Adding support for special characters in SaaS request body (#5099) Co-authored-by: Andres Torres --- CHANGELOG.md | 1 + dev-requirements.txt | 1 + .../connectors/saas/authenticated_client.py | 4 ++ src/fides/api/util/logger_context_utils.py | 7 ++- .../saas/test_onesignal_task.py | 5 +- .../saas/test_authenticated_client.py | 46 ++++++++++++++++++- 6 files changed, 60 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13bef495da8..3092cf7d88d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The types of changes are: ### Added - Add AWS Tags in the meta field for Fides system when using `fides generate` [#4998](https://github.com/ethyca/fides/pull/4998) - Added access and erasure support for Checkr integration [#5121](https://github.com/ethyca/fides/pull/5121) +- Added support for special characters in SaaS request payloads [#5099](https://github.com/ethyca/fides/pull/5099) ### Changed - Moving Privacy Center endpoint logging behind debug flag [#5103](https://github.com/ethyca/fides/pull/5103) diff --git a/dev-requirements.txt b/dev-requirements.txt index b6e1c69ba0a..467a0bfab6b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -23,4 +23,5 @@ types-toml==0.10.8 types-ujson==5.4.0 types-urllib3==1.26.23 watchfiles==0.19.0 +werkzeug==3.0.3 xenon==0.9.0 diff --git a/src/fides/api/service/connectors/saas/authenticated_client.py b/src/fides/api/service/connectors/saas/authenticated_client.py index 8a8626c1109..6d78dfb232d 100644 --- a/src/fides/api/service/connectors/saas/authenticated_client.py +++ b/src/fides/api/service/connectors/saas/authenticated_client.py @@ -219,6 +219,10 @@ def send( # extract the hostname from the complete URL and verify its safety deny_unsafe_hosts(urlparse(prepared_request.url).netloc) + # utf-8 encode the body before sending + if isinstance(prepared_request.body, str): + prepared_request.body = prepared_request.body.encode("utf-8") + response = self.session.send(prepared_request) ignore_error = self._should_ignore_error( diff --git a/src/fides/api/util/logger_context_utils.py b/src/fides/api/util/logger_context_utils.py index 2d86507a4ca..7ad80b74d43 100644 --- a/src/fides/api/util/logger_context_utils.py +++ b/src/fides/api/util/logger_context_utils.py @@ -103,7 +103,12 @@ def request_details( LoggerContextKeys.url.value: prepared_request.url, } if CONFIG.dev_mode and prepared_request.body is not None: - details[LoggerContextKeys.body.value] = prepared_request.body + if isinstance(prepared_request.body, bytes): + details[LoggerContextKeys.body.value] = prepared_request.body.decode( + "utf-8" + ) + elif isinstance(prepared_request.body, str): + details[LoggerContextKeys.body.value] = prepared_request.body if response is not None: if CONFIG.dev_mode and response.content: diff --git a/tests/ops/integration_tests/saas/test_onesignal_task.py b/tests/ops/integration_tests/saas/test_onesignal_task.py index 42aed62d66d..15f13ae0a36 100644 --- a/tests/ops/integration_tests/saas/test_onesignal_task.py +++ b/tests/ops/integration_tests/saas/test_onesignal_task.py @@ -5,6 +5,7 @@ from tests.ops.test_helpers.saas_test_utils import poll_for_existence +@pytest.mark.skip(reason="Temporarily disabled test") @pytest.mark.integration_saas class TestOneSignalConnector: def test_connection(self, onesignal_runner: ConnectorRunner): @@ -24,7 +25,7 @@ async def test_access_request( ): request.getfixturevalue(dsr_version) # REQUIRED to test both DSR 3.0 and 2.0 - access_results = await onesignal_runner.access_request( + await onesignal_runner.access_request( access_policy=policy, identities={"email": onesignal_identity_email} ) @@ -45,7 +46,7 @@ async def test_non_strict_erasure_request( player_id = onesignal_erasure_data ( - access_results, + _, erasure_results, ) = await onesignal_runner.non_strict_erasure_request( access_policy=policy, diff --git a/tests/ops/service/connectors/saas/test_authenticated_client.py b/tests/ops/service/connectors/saas/test_authenticated_client.py index 687f618ef1d..88242aa2b64 100644 --- a/tests/ops/service/connectors/saas/test_authenticated_client.py +++ b/tests/ops/service/connectors/saas/test_authenticated_client.py @@ -1,10 +1,14 @@ +import threading import time import unittest.mock as mock from email.utils import formatdate -from typing import Any, Dict +from typing import Any, Dict, Generator import pytest +from loguru import logger from requests import ConnectionError, Response, Session +from werkzeug.serving import make_server +from werkzeug.wrappers import Response as WerkzeugResponse from fides.api.common_exceptions import ClientUnsuccessfulException, ConnectionException from fides.api.models.connectionconfig import ConnectionConfig, ConnectionType @@ -59,6 +63,33 @@ def test_authenticated_client( ) +@pytest.fixture +def test_http_server() -> Generator[str, None, None]: + """ + Creates a simple HTTP server for testing purposes. + + This fixture sets up a Werkzeug server running on localhost with a + dynamically assigned port. The server responds to all requests with + a "Request received" message. + + The server is automatically shut down after the test is complete. + """ + + def simple_app(environ, start_response): + logger.info("Request received") + response = WerkzeugResponse("Request received") + return response(environ, start_response) + + server = make_server("localhost", 0, simple_app) + server_thread = threading.Thread(target=server.serve_forever) + server_thread.start() + + yield f"http://{server.server_address[0]}:{server.server_address[1]}" + + server.shutdown() + server_thread.join() + + @pytest.mark.unit_saas class TestAuthenticatedClient: @mock.patch.object(Session, "send") @@ -145,6 +176,19 @@ def test_client_ignores_errors( errors_to_ignore=[401], ) + def test_sending_special_characters( + self, test_authenticated_client, test_http_server + ): + request_params = SaaSRequestParams( + method=HTTPMethod.POST, + path="/", + body='{"addr": "1234 Peterson’s Farm Rd."}', + headers={"Content-Type": "application/json"}, + ) + + test_authenticated_client.uri = test_http_server + test_authenticated_client.send(request_params) + @pytest.mark.unit_saas class TestRetryAfterHeaderParsing: