From d5dce5c4969163aa1fb6a8fa24d7c11999835ce6 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Mon, 28 Nov 2022 10:14:22 -0500 Subject: [PATCH] Provide test coverage for fides connector Clean up for mypy and linting --- .../remote_fides_example_test_dataset.yml | 13 + docker-compose.child-env.yml | 98 ++++ .../docs/development/contributing_details.md | 2 +- noxfiles/dev_nox.py | 11 + scripts/load_fides_child_examples.py | 37 ++ scripts/setup/fides_connector.py | 38 ++ .../v1/endpoints/connection_type_endpoints.py | 1 + .../connection_configuration/__init__.py | 4 +- .../connection_secrets_fides.py | 15 +- .../ops/service/connectors/fides/__init__.py | 0 .../service/connectors/fides/fides_client.py | 53 ++- .../ops/service/connectors/fides_connector.py | 42 +- .../test_connection_template_endpoints.py | 4 +- .../v1/endpoints/test_dataset_endpoints.py | 6 +- tests/ops/conftest.py | 1 + tests/ops/fixtures/application_fixtures.py | 6 + .../fides_connector_example_fixtures.py | 95 ++++ tests/ops/integration_test_config.toml | 5 + .../ops/service/connectors/fides/__init__.py | 0 .../connectors/fides/test_fides_client.py | 427 ++++++++++++++++++ .../connectors/test_fides_connector.py | 101 +++++ 21 files changed, 906 insertions(+), 53 deletions(-) create mode 100644 data/dataset/remote_fides_example_test_dataset.yml create mode 100644 docker-compose.child-env.yml create mode 100644 scripts/load_fides_child_examples.py create mode 100644 scripts/setup/fides_connector.py create mode 100644 src/fides/api/ops/service/connectors/fides/__init__.py create mode 100644 tests/ops/fixtures/fides_connector_example_fixtures.py create mode 100644 tests/ops/service/connectors/fides/__init__.py create mode 100644 tests/ops/service/connectors/fides/test_fides_client.py create mode 100644 tests/ops/service/connectors/test_fides_connector.py diff --git a/data/dataset/remote_fides_example_test_dataset.yml b/data/dataset/remote_fides_example_test_dataset.yml new file mode 100644 index 00000000000..672a45ba206 --- /dev/null +++ b/data/dataset/remote_fides_example_test_dataset.yml @@ -0,0 +1,13 @@ +dataset: + - fides_key: fides_example_child_test_dataset + name: Fides Child Example Test Dataset + description: Example of a Remote Fides Child Dataset + collections: + - name: privacy_request + fields: + - name: placeholder + data_categories: [system.operations] + fidesops_meta: + identity: email + - name: id # this is intented to store the resulting privacy request ID + data_categories: [user.contact.email] \ No newline at end of file diff --git a/docker-compose.child-env.yml b/docker-compose.child-env.yml new file mode 100644 index 00000000000..319afdd363c --- /dev/null +++ b/docker-compose.child-env.yml @@ -0,0 +1,98 @@ +services: + fides-child: + image: ethyca/fides:local + command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src fides.api.main:app + healthcheck: + test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ] + interval: 20s + timeout: 5s + retries: 10 + ports: + - "8081:8080" + depends_on: + fides-child-db: + condition: service_healthy + redis-child: + condition: service_started + expose: + - 8080 + env_file: + - .env + environment: + FIDES__CONFIG_PATH: ${FIDES__CONFIG_PATH:-/fides/.fides/fides.toml} + FIDES__CLI__ANALYTICS_ID: ${FIDES__CLI__ANALYTICS_ID:-} + FIDES__CLI__SERVER_HOST: "fides-child" + FIDES__CLI__SERVER_PORT: "8080" + FIDES__DATABASE__SERVER: "fides-child-db" + FIDES__DEV_MODE: "True" + FIDES__REDIS__ENABLED: "True" + FIDES__REDIS__HOST: "redis-child" + FIDES__TEST_MODE: "True" + FIDES__USER__ANALYTICS_OPT_OUT: "True" + VAULT_ADDR: ${VAULT_ADDR:-} + VAULT_NAMESPACE: ${VAULT_NAMESPACE:-} + VAULT_TOKEN: ${VAULT_TOKEN:-} + FIDES__SECURITY__PARENT_SERVER_USERNAME: parent + FIDES__SECURITY__PARENT_SERVER_PASSWORD: parentpassword1! + volumes: + - type: bind + source: . + target: /fides + read_only: False + + fides-child-ui: + image: ethyca/fides:local-ui + command: npm run dev + expose: + - 3000 + ports: + - "3002:3000" + volumes: + - type: bind + source: . + target: /fides + read_only: False + # do not volume mount over the node_modules + - /fides/clients/admin-ui/node_modules + environment: + - NEXT_PUBLIC_FIDESCTL_API_SERVER=http://fides-child:8080 + - NEXT_PUBLIC_FIDESOPS_API=http://fides-child:8080 + + fides-child-db: + image: postgres:12 + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U postgres" ] + interval: 15s + timeout: 5s + retries: 5 + volumes: + # TODO - can we update this to not share the volume mount with primary Fides? + - postgres:/var/lib/postgresql/data + expose: + - 5432 + ports: + - "5433:5432" + environment: + POSTGRES_USER: "postgres" + POSTGRES_PASSWORD: "fides" + POSTGRES_DB: "fides" + deploy: + placement: + constraints: + - node.labels.fides.app-db-data == true + + redis-child: + image: "redis:6.2.5-alpine" + command: redis-server --requirepass testpassword + environment: + - REDIS_PASSWORD=testpassword + expose: + - 6379 + ports: + - "0.0.0.0:6380:6379" + +volumes: + postgres: null + +networks: + fides_network: diff --git a/docs/fides/docs/development/contributing_details.md b/docs/fides/docs/development/contributing_details.md index 8d7c78b6687..485fdd54f94 100644 --- a/docs/fides/docs/development/contributing_details.md +++ b/docs/fides/docs/development/contributing_details.md @@ -93,7 +93,7 @@ Some common Alembic commands are listed below. For a comprehensive guide see: None: datastore for datastore in session.posargs if datastore in ALL_DATASTORES ] or None + if "child" in session.posargs: + session.run( + "docker", + "compose", + "-f", + "docker-compose.child-env.yml", + "up", + "-d", + external=True, + ) + if "ui" in session.posargs: build(session, "admin_ui") session.run("docker", "compose", "up", "-d", "fides-ui", external=True) diff --git a/scripts/load_fides_child_examples.py b/scripts/load_fides_child_examples.py new file mode 100644 index 00000000000..da40878864d --- /dev/null +++ b/scripts/load_fides_child_examples.py @@ -0,0 +1,37 @@ +""" +This script is used to seed the application database with example +Fides connector configurations to integration test Fides parent-child interactions. + +This script is only designed to be run along with the `child` Nox flag. +""" + +from setup import constants +from setup.authentication import get_auth_header +from setup.fides_connector import create_fides_connector +from setup.healthcheck import check_health +from setup.user import create_user + +print("Generating example data for local Fides - child test environment...") + +try: + check_health() +except RuntimeError: + print( + f"Connection error. Please ensure Fides is healthy and running at {constants.FIDES_URL}." + ) + raise + +# Start by creating an OAuth client and user for testing +auth_header = get_auth_header() +create_user( + auth_header=auth_header, +) + + +# Configure a connector to the example Postgres database +create_fides_connector( + auth_header=auth_header, +) + + +print("Examples loaded successfully!") diff --git a/scripts/setup/fides_connector.py b/scripts/setup/fides_connector.py new file mode 100644 index 00000000000..fae26db5db1 --- /dev/null +++ b/scripts/setup/fides_connector.py @@ -0,0 +1,38 @@ +import logging +from typing import Dict + +from setup.database_connector import ( + create_database_connector, + update_database_connector_secrets, +) +from setup.dataset import create_dataset + +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +def create_fides_connector( + auth_header: Dict[str, str], + key: str = "fides_connector", + verify: bool = True, +): + create_database_connector( + auth_header=auth_header, + key=key, + connection_type="fides", + ) + update_database_connector_secrets( + auth_header=auth_header, + key=key, + secrets={ + "uri": "http://fides-child:8080", + "username": "parent", + "password": "parentpassword1!", + }, + verify=verify, + ) + create_dataset( + auth_header=auth_header, + connection_key=key, + yaml_path="data/dataset/remote_fides_example_test_dataset.yml", + ) diff --git a/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py b/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py index 38fd0192e0d..4d3f7e3ea66 100644 --- a/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py @@ -56,6 +56,7 @@ def is_match(elem: str) -> bool: ConnectionType.manual, ConnectionType.email, ConnectionType.manual_webhook, + ConnectionType.fides, ] and is_match(conn_type.value) ] diff --git a/src/fides/api/ops/schemas/connection_configuration/__init__.py b/src/fides/api/ops/schemas/connection_configuration/__init__.py index 6a5860f9538..b41a9b93d18 100644 --- a/src/fides/api/ops/schemas/connection_configuration/__init__.py +++ b/src/fides/api/ops/schemas/connection_configuration/__init__.py @@ -23,8 +23,8 @@ EmailSchema as EmailSchema, ) from fides.api.ops.schemas.connection_configuration.connection_secrets_fides import ( + FidesConnectorSchema, FidesDocsSchema, - FidesSchema, ) from fides.api.ops.schemas.connection_configuration.connection_secrets_manual_webhook import ( ManualWebhookSchema as ManualWebhookSchema, @@ -105,7 +105,7 @@ ConnectionType.email.value: EmailSchema, ConnectionType.manual_webhook.value: ManualWebhookSchema, ConnectionType.timescale.value: TimescaleSchema, - ConnectionType.fides.value: FidesSchema, + ConnectionType.fides.value: FidesConnectorSchema, } diff --git a/src/fides/api/ops/schemas/connection_configuration/connection_secrets_fides.py b/src/fides/api/ops/schemas/connection_configuration/connection_secrets_fides.py index e0e20f26fc9..1ed1c4cea41 100644 --- a/src/fides/api/ops/schemas/connection_configuration/connection_secrets_fides.py +++ b/src/fides/api/ops/schemas/connection_configuration/connection_secrets_fides.py @@ -5,16 +5,21 @@ ConnectionConfigSecretsSchema, ) +DEFAULT_POLLING_RETRIES: int = 100 +DEFAULT_POLLING_INTERVAL: int = 10 -class FidesSchema(ConnectionConfigSecretsSchema): + +class FidesConnectorSchema(ConnectionConfigSecretsSchema): """Schema to validate the secrets needed to connect to a remote Fides""" - uri: str = None - username: str = None - password: str = None + uri: str + username: str + password: str + polling_retries: int = DEFAULT_POLLING_RETRIES + polling_interval: int = DEFAULT_POLLING_INTERVAL _required_components: List[str] = ["uri", "username", "password"] -class FidesDocsSchema(FidesSchema, NoValidationSchema): +class FidesDocsSchema(FidesConnectorSchema, NoValidationSchema): """Fides Child Secrets Schema for API docs""" diff --git a/src/fides/api/ops/service/connectors/fides/__init__.py b/src/fides/api/ops/service/connectors/fides/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/fides/api/ops/service/connectors/fides/fides_client.py b/src/fides/api/ops/service/connectors/fides/fides_client.py index a53cae0d300..e72c488ba0a 100644 --- a/src/fides/api/ops/service/connectors/fides/fides_client.py +++ b/src/fides/api/ops/service/connectors/fides/fides_client.py @@ -57,8 +57,8 @@ def authenticated_request( self, method: str, path: str, - headers: Dict[str, Any] = {}, - query_params: Dict[str, Any] = {}, + headers: Optional[Dict[str, Any]] = {}, + query_params: Optional[Dict[str, Any]] = {}, data: Optional[Any] = None, json: Optional[Any] = None, ) -> PreparedRequest: @@ -80,14 +80,16 @@ def authenticated_request( return req def create_privacy_request( - self, privacy_request_id: str, identity: Identity, policy_key: str + self, external_id: Optional[str], identity: Identity, policy_key: str ) -> str: """ Create privacy request on remote fides by hitting privacy request endpoint Retruns the created privacy request ID """ pr: PrivacyRequestCreate = PrivacyRequestCreate( - external_id=privacy_request_id, identity=identity, policy_key=policy_key + external_id=external_id, + identity=identity, + policy_key=policy_key, ) request: PreparedRequest = self.authenticated_request( @@ -96,38 +98,36 @@ def create_privacy_request( json=[pr.dict()], ) response = self.session.send(request) - if response.ok: - if response.json()["failed"]: - # TODO better exception here? - raise FidesError( - f"Failed privacy request creation on remote Fides {self.uri} with failure message: {response.json()['failed']['message']}" - ) - return response.json()["succeeded"][0]["id"] - - log.error(f"Error creating privacy request on remote Fides {self.uri}") - response.raise_for_status() - return None + if not response.ok: + log.error(f"Error creating privacy request on remote Fides {self.uri}") + response.raise_for_status() + if response.json()["failed"]: + # TODO better handle errored state here? + raise FidesError( + f"Failed privacy request creation on remote Fides {self.uri} with failure message: {response.json()['failed'][0]['message']}" + ) + return response.json()["succeeded"][0]["id"] def poll_for_request_completion( - self, privacy_request_id: str, retries: int = 0, interval: int = 1 + self, privacy_request_id: str, retries: int, interval: int ) -> dict[str, Any]: """ Poll remote fides for status of privacy request with the given ID until it is complete. This is effectively a blocking call, i.e. it will block the current thread until it determines completion, or until timeout is reached. - Returns storage location, or error + Returns the privacy request record, or error """ while (status := self.request_status(privacy_request_id)[0])[ "status" ] not in COMPLETION_STATUSES: # if we've hit 0, we've run out of retries. # an input arg of 0 is effectively infinite retries + retries -= 1 if retries == 0: raise FidesError( - f"Polling for status of privacy request [{privacy_request_id}] on remote Fides {self.uri} has timed out. Request was last observed with status {status.status}" + f"Polling for status of privacy request [{privacy_request_id}] on remote Fides {self.uri} has timed out. Request was last observed with status {status['status']}" ) - retries -= 1 time.sleep(interval) if status["status"] == PrivacyRequestStatus.error: @@ -152,7 +152,7 @@ def poll_for_request_completion( f"Privacy request [{privacy_request_id}] on remote Fides {self.uri} is in an unknown state. Look at the remote Fides for more information." ) - def request_status(self, privacy_request_id: str = None) -> List: + def request_status(self, privacy_request_id: str = None) -> List[Dict[str, Any]]: """ Return privacy request object that tracks its status """ @@ -164,11 +164,10 @@ def request_status(self, privacy_request_id: str = None) -> List: else None, ) response = self.session.send(request) - if response.ok: - return response.json()["items"] + if not response.ok: + log.error( + f"Error retrieving status of privacy request [{privacy_request_id}] on remote Fides {self.uri}", + ) + response.raise_for_status() - log.error( - f"Error retrieving status of privacy request [{privacy_request_id}] on remote Fides {self.uri}", - ) - response.raise_for_status() - return None + return response.json()["items"] diff --git a/src/fides/api/ops/service/connectors/fides_connector.py b/src/fides/api/ops/service/connectors/fides_connector.py index 59d6ff4a5ed..accea64c60e 100644 --- a/src/fides/api/ops/service/connectors/fides_connector.py +++ b/src/fides/api/ops/service/connectors/fides_connector.py @@ -6,6 +6,9 @@ from fides.api.ops.models.connectionconfig import ConnectionConfig, ConnectionTestStatus from fides.api.ops.models.policy import Policy from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.connection_configuration.connection_secrets_fides import ( + FidesConnectorSchema, +) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.base_connector import BaseConnector from fides.api.ops.service.connectors.fides.fides_client import FidesClient @@ -18,19 +21,21 @@ class FidesConnector(BaseConnector[FidesClient]): def __init__(self, configuration: ConnectionConfig): super().__init__(configuration) + config = FidesConnectorSchema(**self.configuration.secrets or {}) + self.polling_retries = config.polling_retries + self.polling_interval = config.polling_interval def query_config(self, node: TraversalNode) -> QueryConfig[Any]: """Return the query config that corresponds to this connector type""" # no query config for fides connectors - return None def create_client(self) -> FidesClient: """Returns a client used to connect to a Fides instance""" - secrets = self.configuration.secrets + config = FidesConnectorSchema(**self.configuration.secrets or {}) client = FidesClient( - uri=secrets["uri"], - username=secrets["username"], - password=secrets["password"], + uri=config.uri, + username=config.username, + password=config.password, ) client.login() return client @@ -41,7 +46,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: by attempting an authorized API call and ensuring success """ log.info(f"Starting test connection to {self.configuration.key}") - client = self.client() + client: FidesClient = self.client() try: client.request_status() except Exception as e: @@ -59,14 +64,22 @@ def retrieve_data( ) -> List[Row]: """Execute access request and handle response on remote Fides""" - client = self.client() + client: FidesClient = self.client() pr_id: str = client.create_privacy_request( - privacy_request_id=privacy_request.external_id, + external_id=privacy_request.external_id or privacy_request.id, identity=Identity(**privacy_request.get_cached_identity_data()), policy_key=policy.key, ) - return [client.poll_for_request_completion(privacy_request_id=pr_id)] + return [ + { + node.address.value: client.poll_for_request_completion( + privacy_request_id=pr_id, + retries=self.polling_retries, + interval=self.polling_interval, + ) + } + ] def mask_data( self, @@ -77,15 +90,19 @@ def mask_data( input_data: Dict[str, List[Any]], ) -> int: """Execute an erasure request on remote fides""" - client = self.client() + client: FidesClient = self.client() update_ct = 0 for _ in rows: pr_id = client.create_privacy_request( - privacy_request_id=privacy_request.external_id, + external_id=privacy_request.external_id, identity=Identity(**privacy_request.get_cached_identity_data()), policy_key=policy.key, ) - client.poll_for_request_completion(privacy_request_id=pr_id) + client.poll_for_request_completion( + privacy_request_id=pr_id, + retries=self.polling_retries, + interval=self.polling_interval, + ) update_ct += 1 return update_ct @@ -93,4 +110,3 @@ def mask_data( def close(self) -> None: """Close any held resources""" # no held resources for Fides client - pass diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index c41c67d2e26..ab997e67cc5 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -63,8 +63,8 @@ def test_get_connection_types( assert resp.status_code == 200 assert ( len(data) - == len(ConnectionType) + len(saas_template_registry.connector_types()) - 4 - ) # there are 4 connection types that are not returned by the endpoint + == len(ConnectionType) + len(saas_template_registry.connector_types()) - 5 + ) # there are 5 connection types that are not returned by the endpoint assert { "identifier": ConnectionType.postgres.value, diff --git a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py index efb9eacfc97..a2e529076d8 100644 --- a/tests/ops/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_dataset_endpoints.py @@ -483,7 +483,7 @@ def test_patch_datasets_bulk_create( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 10 + assert len(response_body["succeeded"]) == 11 assert len(response_body["failed"]) == 0 # Confirm that postgres dataset matches the values we provided @@ -602,7 +602,7 @@ def test_patch_datasets_bulk_update( assert response.status_code == 200 response_body = json.loads(response.text) - assert len(response_body["succeeded"]) == 10 + assert len(response_body["succeeded"]) == 11 assert len(response_body["failed"]) == 0 # test postgres @@ -821,7 +821,7 @@ def test_patch_datasets_failed_response( assert response.status_code == 200 # Returns 200 regardless response_body = json.loads(response.text) assert len(response_body["succeeded"]) == 0 - assert len(response_body["failed"]) == 10 + assert len(response_body["failed"]) == 11 for failed_response in response_body["failed"]: assert "Dataset create/update failed" in failed_response["message"] diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 0e0dc08cb36..22fcca791e4 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -30,6 +30,7 @@ from .fixtures.application_fixtures import * from .fixtures.bigquery_fixtures import * from .fixtures.email_fixtures import * +from .fixtures.fides_connector_example_fixtures import * from .fixtures.integration_fixtures import * from .fixtures.manual_fixtures import * from .fixtures.manual_webhook_fixtures import * diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index b6ad8034e84..7ead458c160 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -113,6 +113,11 @@ "username": pydash.get(integration_config, "timescale_example.user"), "password": pydash.get(integration_config, "timescale_example.password"), }, + "fides_example": { + "uri": pydash.get(integration_config, "fides_example.uri"), + "username": pydash.get(integration_config, "fides_example.username"), + "password": pydash.get(integration_config, "fides_example.password"), + }, } @@ -1241,6 +1246,7 @@ def example_datasets() -> List[Dict]: "data/dataset/bigquery_example_test_dataset.yml", "data/dataset/manual_dataset.yml", "data/dataset/email_dataset.yml", + "data/dataset/remote_fides_example_test_dataset.yml", ] for filename in example_filenames: example_datasets += load_dataset(filename) diff --git a/tests/ops/fixtures/fides_connector_example_fixtures.py b/tests/ops/fixtures/fides_connector_example_fixtures.py new file mode 100644 index 00000000000..25a84a458fb --- /dev/null +++ b/tests/ops/fixtures/fides_connector_example_fixtures.py @@ -0,0 +1,95 @@ +import logging +from typing import Dict, Generator, List, Tuple +from uuid import uuid4 + +import pytest +from sqlalchemy.orm import Session + +from fides.api.ops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.service.connectors import FidesConnector +from fides.ctl.core.config import get_config + +logger = logging.getLogger(__name__) + +CONFIG = get_config() + +from .application_fixtures import integration_secrets + + +@pytest.fixture(scope="function") +def fides_connector_example_secrets(): + return integration_secrets["fides_example"] + + +@pytest.fixture(scope="function") +def fides_connector_polling_overrides(): + return (200, 20) + + +@pytest.fixture(scope="function") +def fides_connector_connection_config( + db: Session, fides_connector_example_secrets: dict[str, str] +) -> Generator: + connection_config = ConnectionConfig.create( + db=db, + data={ + "name": str(uuid4()), + "key": "fides_connector_connection_1", + "connection_type": ConnectionType.fides, + "access": AccessLevel.write, + "secrets": fides_connector_example_secrets, + "disabled": False, + "description": "Mock fides connector connection", + }, + ) + yield connection_config + connection_config.delete(db) + + +@pytest.fixture(scope="function") +def test_fides_connector( + fides_connector_connection_config: dict[str, str], +) -> FidesConnector: + return FidesConnector(configuration=fides_connector_connection_config) + + +@pytest.fixture(scope="function") +def test_fides_connector_overriden_polling( + fides_connector_connection_config: Dict[str, str], + fides_connector_polling_overrides: Tuple[int, int], +) -> FidesConnector: + fides_connector_connection_config.secrets[ + "polling_retries" + ] = fides_connector_polling_overrides[0] + fides_connector_connection_config.secrets[ + "polling_interval" + ] = fides_connector_polling_overrides[1] + return FidesConnector(configuration=fides_connector_connection_config) + + +@pytest.fixture +def fides_connector_example_test_dataset_config( + connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], +) -> Generator: + fides_connector_dataset = example_datasets[10] + fides_key = fides_connector_dataset["fides_key"] + connection_config.name = fides_key + connection_config.key = fides_key + connection_config.save(db=db) + dataset = DatasetConfig.create( + db=db, + data={ + "connection_config_id": connection_config.id, + "fides_key": fides_key, + "dataset": fides_connector_dataset, + }, + ) + yield dataset + dataset.delete(db=db) diff --git a/tests/ops/integration_test_config.toml b/tests/ops/integration_test_config.toml index d75a71269ee..a2112f71202 100644 --- a/tests/ops/integration_test_config.toml +++ b/tests/ops/integration_test_config.toml @@ -52,3 +52,8 @@ user="postgres" password="postgres" db="timescale_example" port=5432 + +[fides_example] +uri="http://fides:8080" +username="root_user" +password="Testpassword1!" diff --git a/tests/ops/service/connectors/fides/__init__.py b/tests/ops/service/connectors/fides/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/ops/service/connectors/fides/test_fides_client.py b/tests/ops/service/connectors/fides/test_fides_client.py new file mode 100644 index 00000000000..4d1046f168c --- /dev/null +++ b/tests/ops/service/connectors/fides/test_fides_client.py @@ -0,0 +1,427 @@ +import unittest.mock as mock + +import pytest +from requests import HTTPError + +from fides.api.ctl.utils.errors import FidesError +from fides.api.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fides.api.ops.service.connectors.fides.fides_client import FidesClient + +SAMPLE_TOKEN = "SOME_TOKEN" + + +class MockResponse: + """ + A class to mock Fides API responses + """ + + def __init__(self, ok, json_data): + self.ok = ok + self.json_data = json_data + + def json(self): + return self.json_data + + +@pytest.fixture(scope="function") +def test_fides_client( + fides_connector_example_secrets: dict[str:str], +) -> FidesClient: + return FidesClient( + fides_connector_example_secrets["uri"], + fides_connector_example_secrets["username"], + fides_connector_example_secrets["password"], + ) + + +@pytest.fixture(scope="function") +def authenticated_fides_client( + test_fides_client: FidesClient, +) -> FidesClient: + test_fides_client.login() + return test_fides_client + + +@pytest.fixture(scope="function") +def test_fides_client_bad_credentials( + fides_connector_example_secrets: dict[str:str], +) -> FidesClient: + return FidesClient( + fides_connector_example_secrets["uri"], + fides_connector_example_secrets["username"], + "badpassword", + ) + + +@pytest.mark.unit +class TestFidesClientUnit: + """ + Unit tests against functionality in the FidesClient class + """ + + @mock.patch( + "requests.post", + side_effect=[ + MockResponse(True, {"token_data": {"access_token": SAMPLE_TOKEN}}) + ], + ) + def test_authenticated_request(self, mock_login, test_fides_client: FidesClient): + """ + Assert that authenticated request properly assigns auth token to the request + and that request object has basic expected properties + """ + test_fides_client.login() + request = test_fides_client.authenticated_request("GET", path="/testpath") + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "GET" + assert request.url == test_fides_client.uri + "/testpath" + + request = test_fides_client.authenticated_request("get", path="/testpath") + assert request.method == "GET" + assert request.url == test_fides_client.uri + "/testpath" + + request = test_fides_client.authenticated_request( + "GET", path="/testpath", headers={"another_header": "header_value"} + ) + assert request.method == "GET" + assert request.url == test_fides_client.uri + "/testpath" + assert len(request.headers) == 2 + assert "another_header" in request.headers + assert request.headers["another_header"] == "header_value" + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + + request = test_fides_client.authenticated_request("POST", path="/testpath") + assert request.method == "POST" + assert request.url == test_fides_client.uri + "/testpath" + + def test_authenticated_request_not_logged_in(self, test_fides_client: FidesClient): + """ + Assert that authenticated request helper throws an error if client is not logged in + """ + with pytest.raises(FidesError) as exc: + test_fides_client.authenticated_request("GET", path="/testpath") + assert "No token" in str(exc) + + @mock.patch( + "requests.post", + side_effect=[ + MockResponse(True, {"token_data": {"access_token": SAMPLE_TOKEN}}) + ], + ) + def test_authenticated_request_parameters( + self, mock_login, test_fides_client: FidesClient + ): + test_fides_client.login() + + # test query params on GET + request = test_fides_client.authenticated_request( + "GET", + path="/testpath", + query_params={"param1": "value1", "param2": "value2"}, + ) + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "GET" + assert ( + request.url + == test_fides_client.uri + "/testpath?param1=value1¶m2=value2" + ) + + # test form data passed as tuples + request = test_fides_client.authenticated_request( + "POST", + path="/testpath", + query_params={"param1": "value1", "param2": "value2"}, + data=[("key1", "value1"), ("key2", "value2")], + ) + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "POST" + assert ( + request.url + == test_fides_client.uri + "/testpath?param1=value1¶m2=value2" + ) + assert request.body == "key1=value1&key2=value2" + + # test form data passed as dict + request = test_fides_client.authenticated_request( + "POST", + path="/testpath", + query_params={"param1": "value1", "param2": "value2"}, + data={"key1": "value1", "key2": "value2"}, + ) + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "POST" + assert ( + request.url + == test_fides_client.uri + "/testpath?param1=value1¶m2=value2" + ) + assert request.body == "key1=value1&key2=value2" + + # test body passed as string literal + request = test_fides_client.authenticated_request( + "POST", + path="/testpath", + query_params={"param1": "value1", "param2": "value2"}, + data="testbody", + ) + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "POST" + assert ( + request.url + == test_fides_client.uri + "/testpath?param1=value1¶m2=value2" + ) + assert request.body == "testbody" + + # test json body passed as a dict + request = test_fides_client.authenticated_request( + "POST", + path="/testpath", + query_params={"param1": "value1", "param2": "value2"}, + json={"field1": "value1"}, + ) + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "POST" + assert ( + request.url + == test_fides_client.uri + "/testpath?param1=value1¶m2=value2" + ) + assert request.body == b'{"field1": "value1"}' + + # test json body passed as a list + request = test_fides_client.authenticated_request( + "POST", + path="/testpath", + query_params={"param1": "value1", "param2": "value2"}, + json=[{"field1": "value1"}], + ) + assert "Authorization" in request.headers + assert request.headers["Authorization"] == f"Bearer {SAMPLE_TOKEN}" + assert request.method == "POST" + assert ( + request.url + == test_fides_client.uri + "/testpath?param1=value1¶m2=value2" + ) + assert request.body == b'[{"field1": "value1"}]' + + # The below polling tests need to rely on response mocking, since we can't control + # the status of a given privacy request on a remote server. + # As such, they make some assumptions about the status API and are susceptible to + # false positives if the status API changes, and we don't adjust our tests here! + + @mock.patch( + "requests.Session.send", + side_effect=[ + MockResponse( + True, + { + "items": [ + PrivacyRequest(status=PrivacyRequestStatus.complete).__dict__ + ] + }, + ) + ], + ) + def test_poll_for_completion( + self, mock_privacy_request_status, authenticated_fides_client: FidesClient + ): + pr_record = authenticated_fides_client.poll_for_request_completion( + privacy_request_id="some_successful_request", + retries=10, + interval=1, + ) + assert pr_record["status"] == PrivacyRequestStatus.complete.value + assert mock_privacy_request_status.call_count == 1 + + @mock.patch( + "requests.Session.send", + side_effect=[ + MockResponse( + True, + {"items": [PrivacyRequest(status=PrivacyRequestStatus.error).__dict__]}, + ) + ], + ) + def test_poll_for_completion_errored( + self, mock_privacy_request_status, authenticated_fides_client: FidesClient + ): + with pytest.raises(FidesError) as exc: + pr_record = authenticated_fides_client.poll_for_request_completion( + privacy_request_id="some_errored_request", + retries=10, + interval=1, + ) + assert mock_privacy_request_status.call_count == 1 + assert "encountered an error" in str(exc) + + @mock.patch( + "requests.Session.send", + side_effect=[ + MockResponse( + True, + { + "items": [ + PrivacyRequest( + status=PrivacyRequestStatus.in_processing + ).__dict__ + ] + }, + ), + MockResponse( + True, + { + "items": [ + PrivacyRequest( + status=PrivacyRequestStatus.in_processing + ).__dict__ + ] + }, + ), + ], + ) + def test_poll_for_completion_timeout( + self, mock_privacy_request_status, authenticated_fides_client: FidesClient + ): + with pytest.raises(FidesError) as exc: + pr_record = authenticated_fides_client.poll_for_request_completion( + privacy_request_id="some_pending_request", interval=1, retries=2 + ) + assert mock_privacy_request_status.call_count == 2 + assert "has timed out" in str( + exc + ) and PrivacyRequestStatus.pending.value in str(exc) + + @mock.patch( + "requests.Session.send", + side_effect=[ + MockResponse( + True, + { + "items": [ + PrivacyRequest( + status=PrivacyRequestStatus.in_processing + ).__dict__ + ] + }, + ), + MockResponse( + True, + { + "items": [ + PrivacyRequest( + status=PrivacyRequestStatus.in_processing + ).__dict__ + ] + }, + ), + MockResponse( + True, + { + "items": [ + PrivacyRequest(status=PrivacyRequestStatus.complete).__dict__ + ] + }, + ), + ], + ) + def test_poll_for_completion_blocks_till_complete( + self, mock_privacy_request_status, authenticated_fides_client: FidesClient + ): + pr_record = authenticated_fides_client.poll_for_request_completion( + privacy_request_id="some_pending_request", interval=1, retries=3 + ) + assert mock_privacy_request_status.call_count == 3 + assert pr_record["status"] == PrivacyRequestStatus.complete.value + + +@pytest.mark.integration +class TestFidesClientIntegration: + """ + Integration tests against functionality in the FidesClient class + that interacts with a running Fides server. + + These tests rely on a Fides client that is configured to + connect to the main Fides server running in the + docker compose test environment. + + This is not the most realistic use case, but it can be used to verify + the core FidesClient functionality, without relying on more than + one Fides server instance to be running. + """ + + def test_login(self, test_fides_client: FidesClient): + """Tests login works as expected""" + + # to test login specifically, create a client directly + # so that we don't call `create_client()`, which performs + # login as part of initialization + test_fides_client.login() + assert test_fides_client.token is not None + + def test_login_bad_credentials( + self, test_fides_client_bad_credentials: FidesClient + ): + """Tests login fails with bad credentials""" + + # to test login specifically, get the client directly + # so that we don't call `create_client()`, which performs + # login as part of initialization + + with pytest.raises(HTTPError): + test_fides_client_bad_credentials.login() + assert test_fides_client_bad_credentials.token is None + + def test_create_privacy_request( + self, + authenticated_fides_client: FidesClient, + policy, + db, + ): + """ + Test that properly configured fides client can create and execute a valid access privacy request + Inspired by `test_privacy_request_endpoints.TestCreatePrivacyRequest` + """ + pr_id = authenticated_fides_client.create_privacy_request( + external_id="test_external_id", + identity={"email": "test@example.com"}, + policy_key=policy.key, + ) + assert pr_id is not None + pr: PrivacyRequest = PrivacyRequest.get(db=db, object_id=pr_id) + assert pr.external_id == "test_external_id" + assert pr.policy.key == policy.key + assert pr.status is not None + pr.delete(db=db) + + def test_request_status_no_privacy_request( + self, authenticated_fides_client: FidesClient + ): + """ + Test that request status can be called successfully with no + privacy request ID specified. This acts as a basic test to + validate we can successfully hit authenticated endpoints. + """ + + statuses = authenticated_fides_client.request_status() + assert len(statuses) == 0 + + def test_request_status_privacy_request( + self, authenticated_fides_client: FidesClient, policy + ): + pr_id = authenticated_fides_client.create_privacy_request( + external_id="test_external_id", + identity={"email": "test@example.com"}, + policy_key=policy.key, + ) + assert pr_id is not None + statuses = authenticated_fides_client.request_status(privacy_request_id=pr_id) + assert len(statuses) == 1 + # to make this test more robust to any config changes, + # or environment-specific issues, + # let's not assume anything about the status here. + assert statuses[0]["status"] is not None diff --git a/tests/ops/service/connectors/test_fides_connector.py b/tests/ops/service/connectors/test_fides_connector.py new file mode 100644 index 00000000000..483300a01b5 --- /dev/null +++ b/tests/ops/service/connectors/test_fides_connector.py @@ -0,0 +1,101 @@ +import uuid +from typing import Tuple + +import pytest + +from fides.api.ops.graph.traversal import TraversalNode +from fides.api.ops.models.connectionconfig import ConnectionTestStatus +from fides.api.ops.models.policy import Policy +from fides.api.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus +from fides.api.ops.schemas.connection_configuration.connection_secrets_fides import ( + DEFAULT_POLLING_INTERVAL, + DEFAULT_POLLING_RETRIES, +) +from fides.api.ops.service.connectors.fides.fides_client import FidesClient +from fides.api.ops.service.connectors.fides_connector import FidesConnector +from tests.ops.graph.graph_test_util import generate_node + + +@pytest.mark.unit +class TestFidesConnectorUnit: + """ + Unit tests against functionality in the FidesConnector class + """ + + def test_connector_attributes_assigned_defaults( + self, test_fides_connector: FidesConnector + ): + assert test_fides_connector.polling_interval == DEFAULT_POLLING_INTERVAL + assert test_fides_connector.polling_retries == DEFAULT_POLLING_RETRIES + + def test_connector_attributes_assigned( + self, + test_fides_connector_overriden_polling: FidesConnector, + fides_connector_polling_overrides: Tuple[int, int], + ): + assert ( + test_fides_connector_overriden_polling.polling_retries + == fides_connector_polling_overrides[0] + ) + assert ( + test_fides_connector_overriden_polling.polling_interval + == fides_connector_polling_overrides[1] + ) + + def test_create_client(self, test_fides_connector: FidesConnector): + client: FidesClient = test_fides_connector.create_client() + assert client.token is not None + assert client.uri == test_fides_connector.configuration.secrets["uri"] + assert client.username == test_fides_connector.configuration.secrets["username"] + assert client.password == test_fides_connector.configuration.secrets["password"] + + +@pytest.mark.integration +class TestFidesConnectorIntegration: + """ + Integration tests against functionality in the FidesConnector class + that interacts with a running Fides server + + These tests rely on a Fides connector config that is configured to + connect to the main Fides server running in the + docker compose test environment. + + This is not a realistic use case, but it can be used to verify + the core FidesConnector functionality, without relying on more than + one Fides server instance to be running. + """ + + def test_test_connection(self, test_fides_connector: FidesConnector): + assert test_fides_connector.test_connection() == ConnectionTestStatus.succeeded + + def test_retrieve_data( + self, + test_fides_connector: FidesConnector, + integration_postgres_config, # we load this fixture so that our request actually executes + example_datasets, + policy: Policy, + ): + # not working currently - need to look more closely. + # maybe this type of integration would be better with a proper + # integration setup of two separate fides apps running + + pass + + # privacy_request = PrivacyRequest( + # id=f"test_fides_connector_retrieve_data{uuid.uuid4()}", + # policy=policy, + # ) + # privacy_request.cache_identity( + # identity={"email": "customer-1@example.com"}, + # ) + + # fides connector functionality does not really make use of the node + # so we can create just a placehodler + # node = TraversalNode( + # generate_node("fides_dataset", "fides_collection", "test_field") + # ) + + # result = test_fides_connector.retrieve_data( + # node=node, policy=policy, privacy_request=privacy_request, input_data=[] + # ) + # len(result) == 1