Sovrn Email Consent Connector

.fides/db_dataset.yml

@@ -1313,6 +1313,9 @@ dataset:
     - name: consent_preferences
       data_categories: [ system.operations ]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: awaiting_consent_email_send_at
+      data_categories: [ system.operations ]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
   - name: privacyrequesterror
     data_categories: []
     data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

clients/admin-ui/src/features/common/RequestStatusBadge.tsx

@@ -13,6 +13,10 @@ export const statusPropMap: {
     bg: "green.500",
     label: "Completed",
   },
+  awaiting_consent_email_send: {
+    bg: "gray.400",
+    label: "Awaiting Email Send",
+  },
   denied: {
     bg: "red.500",
     label: "Denied",

clients/admin-ui/src/features/datastore-connections/add-connection/constants.ts

@@ -20,6 +20,7 @@ export const CONNECTION_TYPE_FILTER_MAP = new Map<string, ItemOption>([
     },
   ],
   ["3rd party integrations", { value: SystemType.SAAS.toString() }],
+  ["Email connectors", { value: SystemType.EMAIL.toString() }],
   ["Show all", { value: "" }],
 ]);
 
@@ -38,6 +39,10 @@ export const CONNECTOR_PARAMETERS_OPTIONS: ConnectorParameterOption[] = [
       ConfigurationSettings.DSR_CUSTOMIZATION,
     ],
   },
+  {
+    type: SystemType.EMAIL,
+    options: [ConfigurationSettings.CONNECTOR_PARAMETERS],
+  },
   {
     type: SystemType.SAAS,
     options: [

clients/admin-ui/src/features/privacy-requests/types.ts

@@ -1,5 +1,6 @@
 export type PrivacyRequestStatus =
   | "approved"
+  | "awaiting_consent_email_send"
   | "complete"
   | "denied"
   | "error"

clients/admin-ui/src/types/api/models/SystemType.ts

@@ -9,4 +9,5 @@ export enum SystemType {
   SAAS = "saas",
   DATABASE = "database",
   MANUAL = "manual",
+  EMAIL = "email",
 }

clients/privacy-center/types/api/models/ConsentPreferences.ts

@@ -2,7 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { Consent } from './Consent';
+import type { Consent } from "./Consent";
 
 /**
  * Schema for consent preferences.

clients/privacy-center/types/api/models/ConsentPreferencesWithVerificationCode.ts

@@ -2,9 +2,9 @@
 /* tslint:disable */
 /* eslint-disable */
 
-import type { Consent } from './Consent';
-import type { ConsentWithExecutableStatus } from './ConsentWithExecutableStatus';
-import type { Identity } from './Identity';
+import type { Consent } from "./Consent";
+import type { ConsentWithExecutableStatus } from "./ConsentWithExecutableStatus";
+import type { Identity } from "./Identity";
 
 /**
  * Schema for consent preferences including the verification code.

src/fides/api/ctl/migrations/versions/8e198eb13802_add_sovrn_consent_connector.py

@@ -0,0 +1,90 @@
+"""Add sovrn consent connector
+
+Revision ID: 8e198eb13802
+Revises: 5d62bab40b71
+Create Date: 2023-01-27 21:58:23.344582
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "8e198eb13802"
+down_revision = "5d62bab40b71"
+branch_labels = None
+depends_on = None
+import sqlalchemy as sa
+
+
+def upgrade():
+    # Add to ConnectionType enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual', 'email', 'manual_webhook', 'timescale', 'fides', 'sovrn')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")
+
+    # Add new PrivacyRequest.awaiting_consent_email_send_at column
+    op.add_column(
+        "privacyrequest",
+        sa.Column(
+            "awaiting_consent_email_send_at", sa.DateTime(timezone=True), nullable=True
+        ),
+    )
+
+    # Add ljt_reader_id for sovrn
+    op.execute("ALTER TYPE providedidentitytype ADD VALUE 'ljt_readerID'")
+
+    # Add new privacyrequeststatus
+    op.execute(
+        "alter type privacyrequeststatus add value 'awaiting_consent_email_send'"
+    )
+
+
+def downgrade():
+    # Remove sovrn from the connectiontype enum
+    op.execute("alter type connectiontype rename to connectiontype_old")
+    op.execute(
+        "create type connectiontype as enum('postgres', 'mongodb', 'mysql', 'https', 'snowflake', 'redshift', 'mssql', 'mariadb', 'bigquery', 'saas', 'manual', 'email', 'manual_webhook', 'timescale', 'fides')"
+    )
+    op.execute(
+        (
+            "alter table connectionconfig alter column connection_type type connectiontype using "
+            "connection_type::text::connectiontype"
+        )
+    )
+    op.execute("drop type connectiontype_old")
+
+    # # Drop PrivacyRequest.awaiting_consent_email_send_at column
+    op.drop_column("privacyrequest", "awaiting_consent_email_send_at")
+
+    # Remove ljt_reader_id from the providedidentitytype enum
+    op.execute("ALTER TYPE providedidentitytype RENAME TO providedidentitytype_old")
+    op.execute(
+        "CREATE TYPE providedidentitytype AS ENUM('email', 'phone_number', 'ga_client_id')"
+    )
+    op.execute(
+        "ALTER TABLE providedidentity ALTER COLUMN field_name TYPE providedidentitytype USING field_name::text::providedidentitytype"
+    )
+    op.execute("DROP TYPE providedidentitytype_old")
+
+    # Removing awaiting_consent_email_send privacyrequeststatus
+    op.execute(
+        "delete from privacyrequest where status in ('awaiting_consent_email_send')"
+    )
+    op.execute("alter type privacyrequeststatus rename to privacyrequeststatus_old")
+    op.execute(
+        "create type privacyrequeststatus as enum('in_processing', 'complete', 'pending', 'error', 'paused', 'approved', 'denied', 'canceled', 'identity_unverified', 'requires_input')"
+    )
+    op.execute(
+        (
+            "alter table privacyrequest alter column status type privacyrequeststatus using "
+            "status::text::privacyrequeststatus"
+        )
+    )
+    op.execute("drop type privacyrequeststatus_old")

src/fides/api/main.py

@@ -61,6 +61,9 @@
 )
 
 # pylint: disable=wildcard-import, unused-wildcard-import
+from fides.api.ops.service.privacy_request.consent_email_batch_service import (
+    initiate_scheduled_batch_consent_email_send,
+)
 from fides.api.ops.service.saas_request.override_implementations import *
 from fides.api.ops.tasks.scheduled.scheduler import scheduler
 from fides.api.ops.util.cache import get_cache
@@ -269,6 +272,8 @@ async def setup_server() -> None:
     if not scheduler.running:
         scheduler.start()
 
+    initiate_scheduled_batch_consent_email_send()
+
     logger.debug("Sending startup analytics events...")
     await send_analytics_event(
         AnalyticsEvent(

src/fides/api/ops/email_templates/get_email_template.py

@@ -5,6 +5,7 @@
 
 from fides.api.ops.common_exceptions import EmailTemplateUnhandledActionType
 from fides.api.ops.email_templates.template_names import (
+    CONSENT_REQUEST_EMAIL_FULFILLMENT,
     CONSENT_REQUEST_VERIFICATION_TEMPLATE,
     EMAIL_ERASURE_REQUEST_FULFILLMENT,
     PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE,
@@ -35,6 +36,8 @@ def get_email_template(  # pylint: disable=too-many-return-statements
         return template_env.get_template(SUBJECT_IDENTITY_VERIFICATION_TEMPLATE)
     if action_type == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT:
         return template_env.get_template(EMAIL_ERASURE_REQUEST_FULFILLMENT)
+    if action_type == MessagingActionType.CONSENT_REQUEST_EMAIL_FULFILLMENT:
+        return template_env.get_template(CONSENT_REQUEST_EMAIL_FULFILLMENT)
     if action_type == MessagingActionType.PRIVACY_REQUEST_RECEIPT:
         return template_env.get_template(PRIVACY_REQUEST_RECEIPT_TEMPLATE)
     if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS:

src/fides/api/ops/email_templates/template_names.py

@@ -1,4 +1,5 @@
 CONSENT_REQUEST_VERIFICATION_TEMPLATE = "consent_request_verification.html"
+CONSENT_REQUEST_EMAIL_FULFILLMENT = "consent_request_email_fulfillment.html"
 SUBJECT_IDENTITY_VERIFICATION_TEMPLATE = "subject_identity_verification.html"
 EMAIL_ERASURE_REQUEST_FULFILLMENT = "message_request_email_fulfillment.html"
 PRIVACY_REQUEST_RECEIPT_TEMPLATE = "privacy_request_receipt.html"

src/fides/api/ops/email_templates/templates/consent_request_email_fulfillment.html

@@ -0,0 +1,49 @@
+<!DOCTYPE html>
+<html lang="en">
+   <head>
+      <meta charset="UTF-8">
+      <title>Notification of users' consent preference changes from {{body.controller}}</title>
+      <style>
+         .consent_preferences {
+           padding: 5px;
+           border-bottom: 1px solid #121439;
+           text-align: left;
+         }
+         .identity_column {
+           padding-right: 15px;
+         }
+      </style>
+   </head>
+   <body>
+      <main>
+         <p> The following users of {{body.controller}} have made changes to their consent preferences. You are notified of the changes because
+            {{body.third_party_vendor_name}} has been identified as a third-party processor to {{body.controller}} that processes user information. </p>
+
+         <p> Please find below the updated list of users and their consent preferences:
+            <table>
+               <tr>
+                 {% for identity_type in body.required_identities -%}
+                     <th class="identity_column"> {{identity_type}}</th>
+                 {%- endfor %}
+                 <th>Preferences</th>
+               </tr>
+               {% for requested_change in body.requested_changes -%}
+                  <tr class="consent_preferences">
+                     {% for identity_type in body.required_identities -%}
+                        <td class="identity_column"> {{requested_change.identities.get(identity_type)}}</td>
+                     {%- endfor %}
+                     <td>
+                        {% for consent_option in requested_change.consent_preferences -%}
+                           {{consent_option.data_use}}: {{"Opt-in" if consent_option.opt_in else "Opt-out"}}{{", " if loop.index < requested_change.consent_preferences|length else ""}}
+                        {%- endfor %}
+                     </td>
+                  </tr>
+               {%- endfor %}
+            </table>
+         </p>
+
+         <p> You are legally obligated to honor the users' consent preferences. </p>
+
+      </main>
+   </body>
+</html>

src/fides/api/ops/models/connectionconfig.py

@@ -49,6 +49,7 @@ class ConnectionType(enum.Enum):
     bigquery = "bigquery"
     manual = "manual"  # Run as part of the traversal
     email = "email"
+    sovrn = "sovrn"
     manual_webhook = "manual_webhook"  # Run before the traversal
     timescale = "timescale"
     fides = "fides"
@@ -74,6 +75,7 @@ def human_readable(self) -> str:
             ConnectionType.manual_webhook.value: "Manual Webhook",
             ConnectionType.timescale.value: "TimescaleDB",
             ConnectionType.fides.value: "Fides Connector",
+            ConnectionType.sovrn.value: "Sovrn",
         }
         try:
             return readable_mapping[self.value]

src/fides/api/ops/models/policy.py

@@ -40,6 +40,7 @@ class CurrentStep(EnumType):
     erasure = "erasure"
     consent = "consent"
     erasure_email_post_send = "erasure_email_post_send"
+    consent_email_post_send = "consent_email_post_send"
     post_webhooks = "post_webhooks"
 
 

src/fides/api/ops/models/privacy_request.py

@@ -80,6 +80,7 @@
     CurrentStep.erasure,
     CurrentStep.consent,
     CurrentStep.erasure_email_post_send,
+    CurrentStep.consent_email_post_send,
     CurrentStep.post_webhooks,
 ]
 
@@ -129,6 +130,7 @@ class PrivacyRequestStatus(str, EnumType):
     in_processing = "in_processing"
     complete = "complete"
     paused = "paused"
+    awaiting_consent_email_send = "awaiting_consent_email_send"
     canceled = "canceled"
     error = "error"
 
@@ -225,6 +227,7 @@ class PrivacyRequest(IdentityVerificationMixin, Base):  # pylint: disable=R0904
     paused_at = Column(DateTime(timezone=True), nullable=True)
     identity_verified_at = Column(DateTime(timezone=True), nullable=True)
     due_date = Column(DateTime(timezone=True), nullable=True)
+    awaiting_consent_email_send_at = Column(DateTime(timezone=True), nullable=True)
 
     @property
     def days_left(self: PrivacyRequest) -> Union[int, None]:
@@ -697,6 +700,13 @@ def pause_processing(self, db: Session) -> None:
             },
         )
 
+    def pause_processing_for_consent_email_send(self, db: Session) -> None:
+        """Put the privacy request in a state of awaiting_consent_email_send"""
+        if self.awaiting_consent_email_send_at is None:
+            self.awaiting_consent_email_send_at = datetime.utcnow()
+        self.status = PrivacyRequestStatus.awaiting_consent_email_send
+        self.save(db=db)
+
     def cancel_processing(self, db: Session, cancel_reason: Optional[str]) -> None:
         """Cancels a privacy request.  Currently should only cancel 'pending' tasks"""
         if self.canceled_at is None:
@@ -768,6 +778,7 @@ class ProvidedIdentityType(EnumType):
     email = "email"
     phone_number = "phone_number"
     ga_client_id = "ga_client_id"
+    ljt_readerID = "ljt_readerID"
 
 
 class ProvidedIdentity(Base):  # pylint: disable=R0904

src/fides/api/ops/schemas/connection_configuration/__init__.py

@@ -22,6 +22,15 @@
 from fides.api.ops.schemas.connection_configuration.connection_secrets_email import (
     EmailSchema as EmailSchema,
 )
+from fides.api.ops.schemas.connection_configuration.connection_secrets_email_consent import (
+    SVORN_REQUIRED_IDENTITY as SVORN_REQUIRED_IDENTITY,
+)
+from fides.api.ops.schemas.connection_configuration.connection_secrets_email_consent import (
+    ConsentEmailDocsSchema as ConsentEmailDocsSchema,
+)
+from fides.api.ops.schemas.connection_configuration.connection_secrets_email_consent import (
+    ConsentEmailSchema as ConsentEmailSchema,
+)
 from fides.api.ops.schemas.connection_configuration.connection_secrets_fides import (
     FidesConnectorSchema,
     FidesDocsSchema,
@@ -80,6 +89,9 @@
 from fides.api.ops.schemas.connection_configuration.connection_secrets_snowflake import (
     SnowflakeSchema as SnowflakeSchema,
 )
+from fides.api.ops.schemas.connection_configuration.connection_secrets_sovrn import (
+    SovrnEmailSchema,
+)
 from fides.api.ops.schemas.connection_configuration.connection_secrets_timescale import (
     TimescaleDocsSchema as TimescaleDocsSchema,
 )
@@ -106,6 +118,7 @@
     ConnectionType.manual_webhook.value: ManualWebhookSchema,
     ConnectionType.timescale.value: TimescaleSchema,
     ConnectionType.fides.value: FidesConnectorSchema,
+    ConnectionType.sovrn.value: SovrnEmailSchema,
 }
 
 
@@ -149,4 +162,5 @@ def get_connection_secrets_schema(
     ManualWebhookSchemaforDocs,
     TimescaleDocsSchema,
     FidesDocsSchema,
+    ConsentEmailDocsSchema,
 ]

src/fides/api/ops/schemas/connection_configuration/connection_config.py

@@ -64,6 +64,7 @@ class SystemType(Enum):
     saas = "saas"
     database = "database"
     manual = "manual"
+    email = "email"
 
 
 class ConnectionSystemTypeMap(BaseModel):