Provide test coverage for fides connector

Clean up for mypy and linting

data/dataset/remote_fides_example_test_dataset.yml

@@ -0,0 +1,13 @@
+dataset:
+  - fides_key: fides_example_child_test_dataset
+    name: Fides Child Example Test Dataset
+    description: Example of a Remote Fides Child Dataset
+    collections:
+      - name: privacy_request
+        fields:
+          - name: placeholder
+            data_categories: [system.operations]
+            fidesops_meta:
+              identity: email
+          - name: id # this is intented to store the resulting privacy request ID
+            data_categories: [user.contact.email]

docker-compose.child-env.yml

@@ -0,0 +1,98 @@
+services:
+  fides-child:
+    image: ethyca/fides:local
+    command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src fides.api.main:app
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ]
+      interval: 20s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "8081:8080"
+    depends_on:
+      fides-child-db:
+        condition: service_healthy
+      redis-child:
+        condition: service_started
+    expose:
+      - 8080
+    env_file:
+      - .env
+    environment:
+      FIDES__CONFIG_PATH: ${FIDES__CONFIG_PATH:-/fides/.fides/fides.toml}
+      FIDES__CLI__ANALYTICS_ID: ${FIDES__CLI__ANALYTICS_ID:-}
+      FIDES__CLI__SERVER_HOST: "fides-child"
+      FIDES__CLI__SERVER_PORT: "8080"
+      FIDES__DATABASE__SERVER: "fides-child-db"
+      FIDES__DEV_MODE: "True"
+      FIDES__REDIS__ENABLED: "True"
+      FIDES__REDIS__HOST: "redis-child"
+      FIDES__TEST_MODE: "True"
+      FIDES__USER__ANALYTICS_OPT_OUT: "True"
+      VAULT_ADDR: ${VAULT_ADDR:-}
+      VAULT_NAMESPACE: ${VAULT_NAMESPACE:-}
+      VAULT_TOKEN: ${VAULT_TOKEN:-}
+      FIDES__SECURITY__PARENT_SERVER_USERNAME: parent
+      FIDES__SECURITY__PARENT_SERVER_PASSWORD: parentpassword1!
+    volumes:
+      - type: bind
+        source: .
+        target: /fides
+        read_only: False
+
+  fides-child-ui:
+    image: ethyca/fides:local-ui
+    command: npm run dev
+    expose:
+      - 3000
+    ports:
+      - "3002:3000"
+    volumes:
+      - type: bind
+        source: .
+        target: /fides
+        read_only: False
+      # do not volume mount over the node_modules
+      - /fides/clients/admin-ui/node_modules
+    environment:
+      - NEXT_PUBLIC_FIDESCTL_API_SERVER=http://fides-child:8080
+      - NEXT_PUBLIC_FIDESOPS_API=http://fides-child:8080
+
+  fides-child-db:
+    image: postgres:12
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+    volumes:
+      # TODO - can we update this to not share the volume mount with primary Fides?
+      - postgres:/var/lib/postgresql/data
+    expose:
+      - 5432
+    ports:
+      - "5433:5432"
+    environment:
+      POSTGRES_USER: "postgres"
+      POSTGRES_PASSWORD: "fides"
+      POSTGRES_DB: "fides"
+    deploy:
+      placement:
+        constraints:
+          - node.labels.fides.app-db-data == true
+
+  redis-child:
+    image: "redis:6.2.5-alpine"
+    command: redis-server --requirepass testpassword
+    environment:
+      - REDIS_PASSWORD=testpassword
+    expose:
+      - 6379
+    ports:
+      - "0.0.0.0:6380:6379"
+
+volumes:
+  postgres: null
+
+networks:
+  fides_network:

docs/fides/docs/development/contributing_details.md

@@ -93,7 +93,7 @@ Some common Alembic commands are listed below. For a comprehensive guide see: <h
 
 The commands will need to be run inside a shell on your Docker containers, which can be opened with `nox -s dev -- shell`.
 
-In the `/src/fides` directory:
+In the `/src/fides/api/ctl` directory:
 
 - Migrate your database to the latest state: `alembic upgrade head`
 - Get revision id of previous migration: `alembic current`

noxfiles/dev_nox.py

@@ -30,6 +30,17 @@ def dev(session: Session) -> None:
         datastore for datastore in session.posargs if datastore in ALL_DATASTORES
     ] or None
 
+    if "child" in session.posargs:
+        session.run(
+            "docker",
+            "compose",
+            "-f",
+            "docker-compose.child-env.yml",
+            "up",
+            "-d",
+            external=True,
+        )
+
     if "ui" in session.posargs:
         build(session, "admin_ui")
         session.run("docker", "compose", "up", "-d", "fides-ui", external=True)

scripts/load_fides_child_examples.py

@@ -0,0 +1,37 @@
+"""
+This script is used to seed the application database with example
+Fides connector configurations to integration test Fides parent-child interactions.
+
+This script is only designed to be run along with the `child` Nox flag.
+"""
+
+from setup import constants
+from setup.authentication import get_auth_header
+from setup.fides_connector import create_fides_connector
+from setup.healthcheck import check_health
+from setup.user import create_user
+
+print("Generating example data for local Fides - child test environment...")
+
+try:
+    check_health()
+except RuntimeError:
+    print(
+        f"Connection error. Please ensure Fides is healthy and running at {constants.FIDES_URL}."
+    )
+    raise
+
+# Start by creating an OAuth client and user for testing
+auth_header = get_auth_header()
+create_user(
+    auth_header=auth_header,
+)
+
+
+# Configure a connector to the example Postgres database
+create_fides_connector(
+    auth_header=auth_header,
+)
+
+
+print("Examples loaded successfully!")

scripts/setup/fides_connector.py

@@ -0,0 +1,38 @@
+import logging
+from typing import Dict
+
+from setup.database_connector import (
+    create_database_connector,
+    update_database_connector_secrets,
+)
+from setup.dataset import create_dataset
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level=logging.INFO)
+
+
+def create_fides_connector(
+    auth_header: Dict[str, str],
+    key: str = "fides_connector",
+    verify: bool = True,
+):
+    create_database_connector(
+        auth_header=auth_header,
+        key=key,
+        connection_type="fides",
+    )
+    update_database_connector_secrets(
+        auth_header=auth_header,
+        key=key,
+        secrets={
+            "uri": "http://fides-child:8080",
+            "username": "parent",
+            "password": "parentpassword1!",
+        },
+        verify=verify,
+    )
+    create_dataset(
+        auth_header=auth_header,
+        connection_key=key,
+        yaml_path="data/dataset/remote_fides_example_test_dataset.yml",
+    )

src/fides/api/ops/api/v1/endpoints/connection_type_endpoints.py

@@ -56,6 +56,7 @@ def is_match(elem: str) -> bool:
                     ConnectionType.manual,
                     ConnectionType.email,
                     ConnectionType.manual_webhook,
+                    ConnectionType.fides,
                 ]
                 and is_match(conn_type.value)
             ]

src/fides/api/ops/schemas/connection_configuration/__init__.py

@@ -23,8 +23,8 @@
     EmailSchema as EmailSchema,
 )
 from fides.api.ops.schemas.connection_configuration.connection_secrets_fides import (
+    FidesConnectorSchema,
     FidesDocsSchema,
-    FidesSchema,
 )
 from fides.api.ops.schemas.connection_configuration.connection_secrets_manual_webhook import (
     ManualWebhookSchema as ManualWebhookSchema,
@@ -105,7 +105,7 @@
     ConnectionType.email.value: EmailSchema,
     ConnectionType.manual_webhook.value: ManualWebhookSchema,
     ConnectionType.timescale.value: TimescaleSchema,
-    ConnectionType.fides.value: FidesSchema,
+    ConnectionType.fides.value: FidesConnectorSchema,
 }
 
 

src/fides/api/ops/schemas/connection_configuration/connection_secrets_fides.py

@@ -5,16 +5,21 @@
     ConnectionConfigSecretsSchema,
 )
 
+DEFAULT_POLLING_RETRIES: int = 100
+DEFAULT_POLLING_INTERVAL: int = 10
 
-class FidesSchema(ConnectionConfigSecretsSchema):
+
+class FidesConnectorSchema(ConnectionConfigSecretsSchema):
     """Schema to validate the secrets needed to connect to a remote Fides"""
 
-    uri: str = None
-    username: str = None
-    password: str = None
+    uri: str
+    username: str
+    password: str
+    polling_retries: int = DEFAULT_POLLING_RETRIES
+    polling_interval: int = DEFAULT_POLLING_INTERVAL
 
     _required_components: List[str] = ["uri", "username", "password"]
 
 
-class FidesDocsSchema(FidesSchema, NoValidationSchema):
+class FidesDocsSchema(FidesConnectorSchema, NoValidationSchema):
     """Fides Child Secrets Schema for API docs"""

src/fides/api/ops/service/connectors/fides/fides_client.py

@@ -57,8 +57,8 @@ def authenticated_request(
         self,
         method: str,
         path: str,
-        headers: Dict[str, Any] = {},
-        query_params: Dict[str, Any] = {},
+        headers: Optional[Dict[str, Any]] = {},
+        query_params: Optional[Dict[str, Any]] = {},
         data: Optional[Any] = None,
         json: Optional[Any] = None,
     ) -> PreparedRequest:
@@ -80,14 +80,16 @@ def authenticated_request(
         return req
 
     def create_privacy_request(
-        self, privacy_request_id: str, identity: Identity, policy_key: str
+        self, external_id: Optional[str], identity: Identity, policy_key: str
     ) -> str:
         """
         Create privacy request on remote fides by hitting privacy request endpoint
         Retruns the created privacy request ID
         """
         pr: PrivacyRequestCreate = PrivacyRequestCreate(
-            external_id=privacy_request_id, identity=identity, policy_key=policy_key
+            external_id=external_id,
+            identity=identity,
+            policy_key=policy_key,
         )
 
         request: PreparedRequest = self.authenticated_request(
@@ -96,38 +98,36 @@ def create_privacy_request(
             json=[pr.dict()],
         )
         response = self.session.send(request)
-        if response.ok:
-            if response.json()["failed"]:
-                # TODO better exception here?
-                raise FidesError(
-                    f"Failed privacy request creation on remote Fides {self.uri} with failure message: {response.json()['failed']['message']}"
-                )
-            return response.json()["succeeded"][0]["id"]
-
-        log.error(f"Error creating privacy request on remote Fides {self.uri}")
-        response.raise_for_status()
-        return None
+        if not response.ok:
+            log.error(f"Error creating privacy request on remote Fides {self.uri}")
+            response.raise_for_status()
+        if response.json()["failed"]:
+            # TODO better handle errored state here?
+            raise FidesError(
+                f"Failed privacy request creation on remote Fides {self.uri} with failure message: {response.json()['failed'][0]['message']}"
+            )
+        return response.json()["succeeded"][0]["id"]
 
     def poll_for_request_completion(
-        self, privacy_request_id: str, retries: int = 0, interval: int = 1
+        self, privacy_request_id: str, retries: int, interval: int
     ) -> dict[str, Any]:
         """
         Poll remote fides for status of privacy request with the given ID until it is complete.
         This is effectively a blocking call, i.e. it will block the current thread until
         it determines completion, or until timeout is reached.
 
-        Returns storage location, or error
+        Returns the privacy request record, or error
         """
         while (status := self.request_status(privacy_request_id)[0])[
             "status"
         ] not in COMPLETION_STATUSES:
             # if we've hit 0, we've run out of retries.
             # an input arg of 0 is effectively infinite retries
+            retries -= 1
             if retries == 0:
                 raise FidesError(
-                    f"Polling for status of privacy request [{privacy_request_id}] on remote Fides {self.uri} has timed out. Request was last observed with status {status.status}"
+                    f"Polling for status of privacy request [{privacy_request_id}] on remote Fides {self.uri} has timed out. Request was last observed with status {status['status']}"
                 )
-            retries -= 1
             time.sleep(interval)
 
         if status["status"] == PrivacyRequestStatus.error:
@@ -152,7 +152,7 @@ def poll_for_request_completion(
             f"Privacy request [{privacy_request_id}] on remote Fides {self.uri} is in an unknown state. Look at the remote Fides for more information."
         )
 
-    def request_status(self, privacy_request_id: str = None) -> List:
+    def request_status(self, privacy_request_id: str = None) -> List[Dict[str, Any]]:
         """
         Return privacy request object that tracks its status
         """
@@ -164,11 +164,10 @@ def request_status(self, privacy_request_id: str = None) -> List:
             else None,
         )
         response = self.session.send(request)
-        if response.ok:
-            return response.json()["items"]
+        if not response.ok:
+            log.error(
+                f"Error retrieving status of privacy request [{privacy_request_id}] on remote Fides {self.uri}",
+            )
+            response.raise_for_status()
 
-        log.error(
-            f"Error retrieving status of privacy request [{privacy_request_id}] on remote Fides {self.uri}",
-        )
-        response.raise_for_status()
-        return None
+        return response.json()["items"]