Con 59 mg appsflyer connector (#4512)

Co-authored-by: Marc Guimond <[email protected]>
Co-authored-by: Adrian Galvan <[email protected]>

CHANGELOG.md

@@ -17,6 +17,9 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fides/compare/2.27.0...main)
 
+### Added
+- Erasure support for AppsFlyer [#4512](https://github.com/ethyca/fides/pull/4512)
+
 ## [2.27.0](https://github.com/ethyca/fides/compare/2.26.0...2.27.0)
 
 ### Added

data/saas/config/appsflyer_config.yml

@@ -0,0 +1,93 @@
+saas_config:
+  fides_key: <instance_fides_key>
+  name: AppsFlyer
+  type: appsflyer
+  description: A sample schema representing the AppsFlyer connector for Fides
+  user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/appsflyer
+  version: 0.1.0
+
+  connector_params:
+    - name: domain
+      default_value: hq1.appsflyer.com
+      description: The AppsFlyer domain. Default hq1.appsflyer.com
+    - name: api_token
+      label: API token
+      description: AppsFlyer API Token (https://support.appsflyer.com/hc/en-us/articles/360004562377#retrieving-api-tokens)
+
+  external_references:
+    - name: appsflyer_user_id
+      label: AppsFlyer user ID field
+      description: The location in your owned database where you store AppsFlyer IDs
+
+  client_config:
+    protocol: https
+    host: <domain>
+    authentication:
+      strategy: bearer
+      configuration:
+        token: <api_token>
+
+  test_request:
+    method: GET
+    path: /api/mng/apps?capabilities=&limit=1
+
+  endpoints:
+    - name: apps
+      requests:
+        read:
+          method: GET
+          path: /api/mng/apps
+          param_values:
+            - name: email
+              identity: email
+          data_path: data
+          pagination:
+            strategy: link
+            configuration:
+              source: body
+              path: links.next
+    - name: user
+      requests:
+        read:
+          request_override: appsflyer_user_read
+          param_values:
+            - name: app_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: apps.id
+                  direction: from
+            - name: user_id
+              references:
+                - appsflyer_user_id
+        delete:
+          method: POST
+          path: /api/gdpr/v1/opendsr_requests
+          body: |
+            {
+              "subject_request_id": "<uuid>", 
+              "subject_request_type": "erasure", 
+              "submitted_time": "<iso_8601_datetime>", 
+              "subject_identities": [
+                {    
+                  "identity_type": "appsflyer_id",
+                  "identity_value": "<user_id>",
+                  "identity_format": "raw"
+                }
+              ],
+              "api_version":"0.1",
+              "property_id": "<app_id>",
+              "status_callback_urls":[ 
+                "https://examplecontroller.com/opengdpr_callbacks"
+              ]
+            }
+          param_values:
+            - name: user_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: user.id
+                  direction: from
+            - name: app_id
+              references:
+                - dataset: <instance_fides_key>
+                  field: user.app_id
+                  direction: from

data/saas/dataset/appsflyer_dataset.yml

@@ -0,0 +1,14 @@
+dataset:
+  - fides_key: <instance_fides_key>
+    name: appsflyer
+    description: A sample dataset representing the AppsFlyer connector for Fides
+    collections:
+      - name: user
+        fields:
+          - name: id
+            data_categories: [user.unique_id]
+            fidesops_meta:
+              primary_key: True
+          - name: app_id
+            data_categories: [system.operations]
+            fidesops_meta:

src/fides/api/service/connectors/saas_query_config.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
+from datetime import datetime
 from itertools import product
 from typing import Any, Dict, List, Literal, Optional, TypeVar
+from uuid import uuid4
 
 import pydash
 from fideslang.models import FidesDatasetReference
@@ -21,8 +23,10 @@
     ALL_OBJECT_FIELDS,
     CUSTOM_PRIVACY_REQUEST_FIELDS,
     FIDESOPS_GROUPED_INPUTS,
+    ISO_8601_DATETIME,
     MASKED_OBJECT_FIELDS,
     PRIVACY_REQUEST_ID,
+    UUID,
     get_identity,
     unflatten_dict,
 )
@@ -307,6 +311,9 @@ def generate_query(
                 0
             ]
 
+        param_values[UUID] = str(uuid4())
+        param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat()
+
         # map param values to placeholders in path, headers, and query params
         saas_request_params: SaaSRequestParams = saas_util.map_param_values(
             self.action, self.collection_name, self.current_request, param_values  # type: ignore
@@ -403,6 +410,8 @@ def generate_update_param_values(  # pylint: disable=R0914
             param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id
 
         param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields
+        param_values[UUID] = str(uuid4())
+        param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")
 
         # remove any row values for fields marked as read-only, these will be omitted from all update maps
         for field_path, field in self.field_map().items():

src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py

@@ -0,0 +1,36 @@
+from typing import Any, Dict, List
+
+from fides.api.graph.traversal import TraversalNode
+from fides.api.models.policy import Policy
+from fides.api.models.privacy_request import PrivacyRequest
+from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient
+from fides.api.service.saas_request.saas_request_override_factory import (
+    SaaSRequestType,
+    register,
+)
+from fides.api.util.collection_util import Row
+
+
+@register("appsflyer_user_read", [SaaSRequestType.READ])
+def appsflyer_user_read(
+    client: AuthenticatedClient,
+    node: TraversalNode,
+    policy: Policy,
+    privacy_request: PrivacyRequest,
+    input_data: Dict[str, List[Any]],
+    secrets: Dict[str, Any],
+) -> List[Row]:
+    """
+    Gather the full list of applications set up, and retrieve the application name and
+    platform information for use with erasure endpoint.
+    """
+
+    user_ids = input_data.get("user_id")
+    app_ids = input_data.get("app_id")
+
+    users = []
+    for app_id in app_ids:
+        for user_id in user_ids:
+            users.append({"id": user_id, "app_id": app_id})
+
+    return users

src/fides/api/util/saas_util.py

@@ -25,6 +25,8 @@
 MASKED_OBJECT_FIELDS = "masked_object_fields"
 ALL_OBJECT_FIELDS = "all_object_fields"
 CUSTOM_PRIVACY_REQUEST_FIELDS = "custom_privacy_request_fields"
+UUID = "uuid"
+ISO_8601_DATETIME = "iso_8601_datetime"
 
 
 def deny_unsafe_hosts(host: str) -> str:

tests/fixtures/saas/appsflyer_fixtures.py

@@ -0,0 +1,62 @@
+import uuid
+from typing import Any, Dict
+
+import pydash
+import pytest
+
+from tests.ops.integration_tests.saas.connector_runner import (
+    ConnectorRunner,
+    generate_random_email,
+)
+from tests.ops.test_helpers.vault_client import get_secrets
+
+secrets = get_secrets("appsflyer")
+
+
+@pytest.fixture(scope="session")
+def appsflyer_secrets(saas_config) -> Dict[str, Any]:
+    return {
+        "domain": pydash.get(saas_config, "appsflyer.domain") or secrets["domain"],
+        "api_token": pydash.get(saas_config, "appsflyer.api_token")
+        or secrets["api_token"],
+    }
+
+
+@pytest.fixture(scope="session")
+def appsflyer_identity_email(saas_config) -> str:
+    return (
+        pydash.get(saas_config, "appsflyer.identity_email") or secrets["identity_email"]
+    )
+
+
+@pytest.fixture
+def appsflyer_erasure_identity_email() -> str:
+    return generate_random_email()
+
+
+@pytest.fixture
+def appsflyer_external_references() -> Dict[str, Any]:
+    return {"appsflyer_user_id": uuid.uuid4()}
+
+
+@pytest.fixture
+def appsflyer_erasure_external_references() -> Dict[str, Any]:
+    return {"appsflyer_user_id": uuid.uuid4()}
+
+
+@pytest.fixture
+def appsflyer_runner(
+    db,
+    cache,
+    appsflyer_secrets,
+    appsflyer_external_references,
+    appsflyer_erasure_external_references,
+) -> ConnectorRunner:
+    return ConnectorRunner(
+        db,
+        cache,
+        "appsflyer",
+        appsflyer_secrets,
+        external_references=appsflyer_external_references,
+        erasure_external_references=appsflyer_erasure_external_references,
+    )

tests/ops/integration_tests/saas/test_appsflyer_task.py

@@ -0,0 +1,40 @@
+import pytest
+
+from fides.api.models.policy import Policy
+from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner
+
+
+@pytest.mark.integration_saas
+class TestAppsFlyerConnector:
+    def test_connection(self, appsflyer_runner: ConnectorRunner):
+        appsflyer_runner.test_connection()
+
+    async def test_access_request(
+        self, appsflyer_runner: ConnectorRunner, policy, appsflyer_identity_email: str
+    ):
+        access_results = await appsflyer_runner.access_request(
+            access_policy=policy, identities={"email": appsflyer_identity_email}
+        )
+
+        assert len(access_results["appsflyer_instance:user"]) == 10
+
+    async def test_non_strict_erasure_request(
+        self,
+        appsflyer_runner: ConnectorRunner,
+        policy: Policy,
+        erasure_policy_string_rewrite: Policy,
+        appsflyer_erasure_identity_email: str,
+    ):
+        (
+            _,
+            erasure_results,
+        ) = await appsflyer_runner.non_strict_erasure_request(
+            access_policy=policy,
+            erasure_policy=erasure_policy_string_rewrite,
+            identities={"email": appsflyer_erasure_identity_email},
+        )
+        assert erasure_results == {
+            "appsflyer_external_dataset:appsflyer_external_collection": 0,
+            "appsflyer_instance:apps": 0,
+            "appsflyer_instance:user": 10,
+        }