Send email alerts on privacy request failures

.fides/db_dataset.yml

@@ -1277,6 +1277,45 @@ dataset:
     - name: updated_at
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+  - name: privacyrequesterror
+    data_categories: []
+    data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    fields:
+    - name: created_at
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: id
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: message_sent
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: privacy_request_id
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: updated_at
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+  - name: privacyrequestnotifications
+    data_categories: []
+    data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    fields:
+    - name: created_at
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: email
+      description: 'The emails of the user that wish to receive error notifications'
+      data_categories:
+      - user.contact.email
+    - name: id
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: notify_after_failures
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: updated_at
+      data_categories: [system.operations]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
   - name: providedidentity
     data_categories: []
     data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

CHANGELOG.md

@@ -25,6 +25,7 @@ The types of changes are:
 
 ### Added
 
+* Send email alerts on privacy request failures once the specified threshold is reached. [#1793](https://github.com/ethyca/fides/pull/1793)
 * Add authenticated privacy request route. [#1819](https://github.com/ethyca/fides/pull/1819)
 * Enable the onboarding flow [#1836](https://github.com/ethyca/fides/pull/1836)
 * Access and erasure support for Fullstory API [#1821](https://github.com/ethyca/fides/pull/1821)

docs/fides/docs/development/postman/Fides.postman_collection.json

@@ -1,9 +1,8 @@
 {
 	"info": {
-		"_postman_id": "1329eef2-2d7e-462d-b8f0-f185557f144c",
+		"_postman_id": "09d75fa5-5a52-4d25-8e6b-7c36a22d6175",
 		"name": "Fidesops",
-		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json",
-		"_exporter_id": "13396647"
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
 	},
 	"item": [
 		{
@@ -73,7 +72,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "[\n    \"client:create\",\n    \"client:update\",\n    \"client:read\",\n    \"client:delete\",\n    \"config:read\",\n    \"connection_type:read\",\n    \"connection:read\",\n    \"connection:create_or_update\",\n    \"connection:delete\",\n    \"connection:instantiate\",\n    \"consent:read\",\n    \"dataset:create_or_update\",\n    \"dataset:delete\",\n    \"dataset:read\",\n    \"encryption:exec\",\n    \"messaging:create_or_update\",\n    \"messaging:read\",\n    \"messaging:delete\",\n    \"policy:create_or_update\",\n    \"policy:read\",\n    \"policy:delete\",\n    \"privacy-request:read\",\n    \"privacy-request:delete\",\n    \"rule:create_or_update\",\n    \"rule:read\",\n    \"rule:delete\",\n    \"scope:read\",\n    \"storage:create_or_update\",\n    \"storage:delete\",\n    \"storage:read\",\n    \"privacy-request:resume\",\n    \"webhook:create_or_update\",\n    \"webhook:read\",\n    \"webhook:delete\",\n    \"saas_config:create_or_update\",\n    \"saas_config:read\",\n    \"saas_config:delete\",\n    \"privacy-request:review\",\n    \"user:create\",\n    \"user:delete\"\n]",
+							"raw": "[\n    \"client:create\",\n    \"client:update\",\n    \"client:read\",\n    \"client:delete\",\n    \"config:read\",\n    \"connection_type:read\",\n    \"connection:read\",\n    \"connection:create_or_update\",\n    \"connection:delete\",\n    \"connection:instantiate\",\n    \"consent:read\",\n    \"dataset:create_or_update\",\n    \"dataset:delete\",\n    \"dataset:read\",\n    \"encryption:exec\",\n    \"messaging:create_or_update\",\n    \"messaging:read\",\n    \"messaging:delete\",\n    \"policy:create_or_update\",\n    \"policy:read\",\n    \"policy:delete\",\n    \"privacy-request:read\",\n    \"privacy-request:delete\",\n    \"privacy-request-notifications:create_or_update\",\n    \"privacy-request-notifications:read\",\n    \"rule:create_or_update\",\n    \"rule:read\",\n    \"rule:delete\",\n    \"scope:read\",\n    \"storage:create_or_update\",\n    \"storage:delete\",\n    \"storage:read\",\n    \"privacy-request:resume\",\n    \"webhook:create_or_update\",\n    \"webhook:read\",\n    \"webhook:delete\",\n    \"saas_config:create_or_update\",\n    \"saas_config:read\",\n    \"saas_config:delete\",\n    \"privacy-request:review\",\n    \"user:create\",\n    \"user:delete\"\n]",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -946,6 +945,68 @@
 						}
 					},
 					"response": []
+				},
+				{
+					"name": "Get Privacy Request Notification Info",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "GET",
+						"header": [],
+						"url": {
+							"raw": "{{host}}/privacy-request/notification/",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"privacy-request",
+								"notification",
+								""
+							]
+						}
+					},
+					"response": []
+				},
+				{
+					"name": "Insert/Update Privacy Request Notification Info",
+					"request": {
+						"auth": {
+							"type": "bearer",
+							"bearer": [
+								{
+									"key": "token",
+									"value": "{{client_token}}",
+									"type": "string"
+								}
+							]
+						},
+						"method": "PUT",
+						"header": [],
+						"body": {
+							"mode": "raw",
+							"raw": "{\n    \"email_addresses\": [\"[email protected]\", \"[email protected]\"],\n    \"notify_after_failures\": 5\n}"
+						},
+						"url": {
+							"raw": "{{host}}/privacy-request/notification/",
+							"host": [
+								"{{host}}"
+							],
+							"path": [
+								"privacy-request",
+								"notification",
+								""
+							]
+						}
+					},
+					"response": []
 				}
 			]
 		},

src/fides/api/ctl/migrations/versions/1f61c765cd1c_merge_alembic_heads.py

@@ -0,0 +1,24 @@
+"""Merge alembic heads
+
+Revision ID: 1f61c765cd1c
+Revises: 8f84fad4e00b, b72541d79f10
+Create Date: 2022-12-02 17:59:08.490577
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1f61c765cd1c'
+down_revision = ('8f84fad4e00b', 'b72541d79f10')
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

src/fides/api/ctl/migrations/versions/28108b17a99c_add_notifications_for_failed_dsr_.py

@@ -0,0 +1,55 @@
+"""Add notifications for failed DSR requests
+
+Revision ID: 28108b17a99c
+Revises: 8f1a19465239
+Create Date: 2022-11-07 23:56:14.066374
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "28108b17a99c"
+down_revision = "8f1a19465239"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "privacyrequestnotifications",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("email", sa.String(), nullable=False),
+        sa.Column("notify_after_failures", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_privacyrequestnotifications_id"),
+        "privacyrequestnotifications",
+        ["id"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        op.f("ix_privacyrequestnotifications_id"),
+        table_name="privacyrequestnotifications",
+    )
+    op.drop_table("privacyrequestnotifications")
+    # ### end Alembic commands ###

src/fides/api/ctl/migrations/versions/58933b5cc6e8_merge_failed_dsr_and_twilio.py

@@ -0,0 +1,24 @@
+"""Merge failed dsr and twilio
+
+Revision ID: 58933b5cc6e8
+Revises: 179f2bb623ae, 28108b17a99c
+Create Date: 2022-11-14 21:26:49.027809
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "58933b5cc6e8"
+down_revision = ("179f2bb623ae", "28108b17a99c")
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

src/fides/api/ctl/migrations/versions/8f84fad4e00b_add_error_message_tracking.py

@@ -0,0 +1,54 @@
+"""Add error message tracking
+
+Revision ID: 8f84fad4e00b
+Revises: 58933b5cc6e8
+Create Date: 2022-11-15 01:38:28.531640
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "8f84fad4e00b"
+down_revision = "58933b5cc6e8"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "privacyrequesterror",
+        sa.Column("id", sa.String(length=255), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(timezone=True),
+            server_default=sa.text("now()"),
+            nullable=True,
+        ),
+        sa.Column("message_sent", sa.Boolean(), nullable=False),
+        sa.Column("privacy_request_id", sa.String(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["privacy_request_id"],
+            ["privacyrequest.id"],
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_privacyrequesterror_id"), "privacyrequesterror", ["id"], unique=False
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_privacyrequesterror_id"), table_name="privacyrequesterror")
+    op.drop_table("privacyrequesterror")
+    # ### end Alembic commands ###

src/fides/api/ops/api/v1/endpoints/drp_endpoints.py

@@ -32,6 +32,9 @@
 from fides.api.ops.schemas.privacy_request import PrivacyRequestDRPStatusResponse
 from fides.api.ops.schemas.redis_cache import Identity
 from fides.api.ops.service.drp.drp_fidesops_mapper import DrpFidesopsMapper
+from fides.api.ops.service.messaging.message_dispatch_service import (
+    check_and_dispatch_error_notifications,
+)
 from fides.api.ops.service.privacy_request.request_runner_service import (
     queue_privacy_request,
 )
@@ -110,6 +113,8 @@ async def create_drp_privacy_request(
             identity=mapped_identity,
         )
 
+        check_and_dispatch_error_notifications(db=db)
+
         logger.info(
             "Decrypting identity for DRP privacy request %s", privacy_request.id
         )