Executing Consent Requests via the Privacy Request Execution Layer [#…

…2146] (#2125)

Adds the foundation to execute consent requests via the privacy request execution framework. When consent preferences are changed, a PrivacyRequest is created (with a Policy > Consent Rule attached) to propagate those consent preferences to third party services, server-side.

.fides/db_dataset.yml

@@ -1018,6 +1018,10 @@ dataset:
     - name: updated_at
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: privacy_request_id
+      description: 'An optional link to the privacy request if one was created to propagate request preferences'
+      data_categories: [ system.operations ]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
   - name: datasetconfig
     data_categories: []
     data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -1279,6 +1283,9 @@ dataset:
     - name: updated_at
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: consent_preferences
+      data_categories: [ system.operations ]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
   - name: privacyrequesterror
     data_categories: []
     data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

clients/privacy-center/config/config.json

@@ -37,6 +37,7 @@
       "email": "required",
       "phone": "optional"
     },
+    "policy_key": "default_consent_policy",
     "consentOptions": [
       {
         "fidesDataUseKey": "advertising",
@@ -45,7 +46,8 @@
         "url": "https://example.com/privacy#data-sales",
         "default": true,
         "highlight": false,
-        "cookieKeys": ["data_sales"]
+        "cookieKeys": ["data_sales"],
+        "executable": true
       },
       {
         "fidesDataUseKey": "advertising.first_party",
@@ -54,7 +56,8 @@
         "url": "https://example.com/privacy#email-marketing",
         "default": true,
         "highlight": false,
-        "cookieKeys": []
+        "cookieKeys": [],
+        "executable": true
       },
       {
         "fidesDataUseKey": "improve",
@@ -63,7 +66,8 @@
         "url": "https://example.com/privacy#analytics",
         "default": true,
         "highlight": false,
-        "cookieKeys": []
+        "cookieKeys": [],
+        "executable": true
       }
     ]
   }

src/fides/api/ctl/database/seed.py

@@ -41,6 +41,9 @@
 DEFAULT_ERASURE_POLICY_RULE = "default_erasure_policy_rule"
 DEFAULT_ERASURE_MASKING_STRATEGY = "hmac"
 
+DEFAULT_CONSENT_POLICY: str = "default_consent_policy"
+DEFAULT_CONSENT_RULE = "default_consent_rule"
+
 
 def create_or_update_parent_user() -> None:
     with sync_session() as db_session:
@@ -278,6 +281,30 @@ async def load_default_dsr_policies() -> None:
             except KeyOrNameAlreadyExists:
                 # This rule target already exists against the Policy
                 pass
+
+        log.info("Creating: Default Consent Policy")
+        consent_policy = Policy.create_or_update(
+            db=db_session,
+            data={
+                "name": "Default Consent Policy",
+                "key": DEFAULT_CONSENT_POLICY,
+                "execution_timeframe": 45,
+                "client_id": client_id,
+            },
+        )
+
+        log.info("Creating: Default Consent Rule")
+        Rule.create_or_update(
+            db=db_session,
+            data={
+                "action_type": ActionType.consent.value,
+                "name": "Default Consent Rule",
+                "key": DEFAULT_CONSENT_RULE,
+                "policy_id": consent_policy.id,
+                "client_id": client_id,
+            },
+        )
+
         log.info("All Policies & Rules Seeded.")
 
 

src/fides/api/ctl/migrations/versions/de456534dbda_add_privacyrequest_consent_preferences_.py

@@ -0,0 +1,43 @@
+"""add privacyrequest consent preferences and ruleuse table
+
+Revision ID: de456534dbda
+Revises: 3caf11127442
+Create Date: 2023-01-03 22:59:45.144538
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "de456534dbda"
+down_revision = "3caf11127442"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "privacyrequest",
+        sa.Column(
+            "consent_preferences",
+            postgresql.JSONB(astext_type=sa.Text()),
+            nullable=True,
+        ),
+    )
+
+    op.add_column(
+        "consentrequest", sa.Column("privacy_request_id", sa.String(), nullable=True)
+    )
+    op.create_foreign_key(
+        None, "consentrequest", "privacyrequest", ["privacy_request_id"], ["id"]
+    )
+
+
+def downgrade():
+    op.drop_column("privacyrequest", "consent_preferences")
+
+    op.drop_constraint(
+        "consentrequest_privacy_request_id_fkey", "consentrequest", type_="foreignkey"
+    )
+    op.drop_column("consentrequest", "privacy_request_id")

src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
-from typing import Optional
+import json
+from typing import Dict, List, Optional, Tuple, Union
 
 from fastapi import Depends, HTTPException, Security
 from loguru import logger
@@ -15,7 +16,11 @@
     HTTP_500_INTERNAL_SERVER_ERROR,
 )
 
+from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY
 from fides.api.ops.api.deps import get_db
+from fides.api.ops.api.v1.endpoints.privacy_request_endpoints import (
+    create_privacy_request_func,
+)
 from fides.api.ops.api.v1.scope_registry import CONSENT_READ
 from fides.api.ops.api.v1.urn_registry import (
     CONSENT_REQUEST,
@@ -37,14 +42,17 @@
     ProvidedIdentityType,
 )
 from fides.api.ops.schemas.messaging.messaging import MessagingMethod
+from fides.api.ops.schemas.privacy_request import BulkPostPrivacyRequests
 from fides.api.ops.schemas.privacy_request import Consent as ConsentSchema
 from fides.api.ops.schemas.privacy_request import (
     ConsentPreferences,
     ConsentPreferencesWithVerificationCode,
     ConsentRequestResponse,
+    PrivacyRequestCreate,
     VerificationCode,
 )
 from fides.api.ops.schemas.redis_cache import Identity
+from fides.api.ops.schemas.shared_schemas import FidesOpsKey
 from fides.api.ops.service._verification import send_verification_code_to_user
 from fides.api.ops.util.api_router import APIRouter
 from fides.api.ops.util.logger import Pii
@@ -54,6 +62,7 @@
 router = APIRouter(tags=["Consent"], prefix=V1_URL_PREFIX)
 
 CONFIG = get_config()
+CONFIG_JSON_PATH = "clients/privacy-center/config/config.json"
 
 
 @router.post(
@@ -115,7 +124,7 @@ def consent_request_verify(
     data: VerificationCode,
 ) -> ConsentPreferences:
     """Verifies the verification code and returns the current consent preferences if successful."""
-    provided_identity = _get_consent_request_and_provided_identity(
+    _, provided_identity = _get_consent_request_and_provided_identity(
         db=db, consent_request_id=consent_request_id, verification_code=data.code
     )
 
@@ -171,7 +180,7 @@ def get_consent_preferences_no_id(
             "turned off.",
         )
 
-    provided_identity = _get_consent_request_and_provided_identity(
+    _, provided_identity = _get_consent_request_and_provided_identity(
         db=db, consent_request_id=consent_request_id, verification_code=None
     )
 
@@ -216,6 +225,86 @@ def get_consent_preferences(
     return _prepare_consent_preferences(db, identity)
 
 
+def load_executable_consent_options(file_path: str) -> List[str]:
+    """Load customer's consentOptions from the config.json file and filter to return only a list
+    of executable consent options"""
+    with open(file_path, encoding="utf-8") as privacy_center_config_file:
+        privacy_center_config: Dict = json.load(privacy_center_config_file)
+        consent_options: List = privacy_center_config.get("consent", {}).get(
+            "consentOptions", []
+        )
+
+        executable_consent_options: List[str] = []
+        for consent in consent_options:
+            data_use: str = consent.get("fidesDataUseKey")
+            if not data_use:
+                continue
+
+            if consent.get("executable"):
+                executable_consent_options.append(data_use)
+            else:
+                logger.info("Consent option: '{}' is not executable.", data_use)
+
+        return executable_consent_options
+
+
+def queue_privacy_request_to_propagate_consent(
+    db: Session,
+    provided_identity: ProvidedIdentity,
+    policy: Union[FidesOpsKey, str],
+    consent_preferences: ConsentPreferences,
+) -> Optional[BulkPostPrivacyRequests]:
+    """
+    Queue a privacy request to carry out propagating consent preferences server-side to third-party systems.
+
+    Only propagate consent preferences which are considered "executable" by the current system. If none of the
+    consent preferences are executable, no Privacy Request is queued.
+    """
+    identity = Identity()
+    setattr(
+        identity,
+        provided_identity.field_name.value,  # type:ignore[attr-defined]
+        provided_identity.encrypted_value["value"],  # type:ignore[index]
+    )  # Pull the information on the ProvidedIdentity for the ConsentRequest to pass along to create a PrivacyRequest
+
+    executable_consent_options: List[str] = load_executable_consent_options(
+        CONFIG_JSON_PATH
+    )
+    executable_consent_preferences: List[Dict] = [
+        pref.dict()
+        for pref in consent_preferences.consent or []
+        if pref.data_use in executable_consent_options
+    ]
+
+    if not executable_consent_preferences:
+        logger.info(
+            "Skipping propagating consent preferences to third-party services as "
+            "specified consent preferences: {} are not executable.",
+            [pref.data_use for pref in consent_preferences.consent or []],
+        )
+        return None
+
+    privacy_request_results: BulkPostPrivacyRequests = create_privacy_request_func(
+        db=db,
+        data=[
+            PrivacyRequestCreate(
+                identity=identity,
+                policy_key=policy,
+                consent_preferences=executable_consent_preferences,
+            )
+        ],
+        authenticated=True,
+    )
+
+    if privacy_request_results.failed or not privacy_request_results.succeeded:
+        raise HTTPException(
+            status_code=HTTP_400_BAD_REQUEST,
+            detail=privacy_request_results.failed[0].message,
+        )
+
+    return privacy_request_results
+
+
 @router.patch(
     CONSENT_REQUEST_PREFERENCES_WITH_ID,
     status_code=HTTP_200_OK,
@@ -228,7 +317,7 @@ def set_consent_preferences(
     data: ConsentPreferencesWithVerificationCode,
 ) -> ConsentPreferences:
     """Verifies the verification code and saves the user's consent preferences if successful."""
-    provided_identity = _get_consent_request_and_provided_identity(
+    consent_request, provided_identity = _get_consent_request_and_provided_identity(
         db=db,
         consent_request_id=consent_request_id,
         verification_code=data.code,
@@ -258,7 +347,27 @@ def set_consent_preferences(
                     status_code=HTTP_400_BAD_REQUEST, detail=Pii(str(exc))
                 )
 
-    return _prepare_consent_preferences(db, provided_identity)
+    consent_preferences: ConsentPreferences = _prepare_consent_preferences(
+        db, provided_identity
+    )
+
+    # Note: This just queues the PrivacyRequest for processing
+    privacy_request_creation_results: Optional[
+        BulkPostPrivacyRequests
+    ] = queue_privacy_request_to_propagate_consent(
+        db,
+        provided_identity,
+        data.policy_key or DEFAULT_CONSENT_POLICY,
+        consent_preferences,
+    )
+
+    if privacy_request_creation_results:
+        consent_request.privacy_request_id = privacy_request_creation_results.succeeded[
+            0
+        ].id
+        consent_request.save(db=db)
+
+    return consent_preferences
 
 
 def _get_or_create_provided_identity(
@@ -355,7 +464,7 @@ def _get_consent_request_and_provided_identity(
     db: Session,
     consent_request_id: str,
     verification_code: Optional[str],
-) -> ProvidedIdentity:
+) -> Tuple[ConsentRequest, ProvidedIdentity]:
     """Verifies the consent request and verification code, then return the ProvidedIdentity if successful."""
     consent_request = ConsentRequest.get_by_key_or_id(
         db=db, data={"id": consent_request_id}
@@ -389,7 +498,7 @@ def _get_consent_request_and_provided_identity(
             detail="No identity found for consent request id",
         )
 
-    return provided_identity
+    return consent_request, provided_identity
 
 
 def _prepare_consent_preferences(