From 3ffa7e75a4aee96bf1d1300eaf1ead88927ed299 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Mon, 27 Nov 2023 14:11:29 -0500 Subject: [PATCH 01/41] started --- data/saas/config/appsflyer_config.yml | 49 +++++++++++++ data/saas/dataset/appsflyer_dataset.yml | 0 tests/fixtures/saas/appsflyer_fixtures.py | 69 +++++++++++++++++++ .../saas/test_appsflyer_task.py | 51 ++++++++++++++ 4 files changed, 169 insertions(+) create mode 100644 data/saas/config/appsflyer_config.yml create mode 100644 data/saas/dataset/appsflyer_dataset.yml create mode 100644 tests/fixtures/saas/appsflyer_fixtures.py create mode 100644 tests/ops/integration_tests/saas/test_appsflyer_task.py diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml new file mode 100644 index 00000000000..fd98b5dac4e --- /dev/null +++ b/data/saas/config/appsflyer_config.yml @@ -0,0 +1,49 @@ +saas_config: + fides_key: + name: Appsflyer SaaS Config + type: appsflyer + description: A sample schema representing the Appsflyer connector for Fides + version: 0.1.0 + + connector_params: + - name: domain + default_value: hq1.appsflyer.com + - name: test_req_id + + client_config: + protocol: https + host: hq1.appsflyer.com + authentication: + strategy: bearer + configuration: + token: + + endpoints: + - name: erasure + requests: + read: + method: GET + path: /api/gdpr/v1/opendsr_requests + body: > + { "subject_request_id":"f4e5a271-f25e-4107-b681-123456789015", + "subject_request_type":"access", + "submitted_time":"2023-05-15T10:00:00Z", "platform": "android", + "subject_identities":[ { + "identity_type":"android_advertising_id", + "identity_value":"4412345-4963-9234-632156789015", + "identity_format":"raw" } ], "api_version":"0.1", + "property_id":"com.madeup.app", "status_callback_urls":[ + "https://examplecontroller.com/opengdpr_callbacks" ] } + delete: + method: POST + path: /api/gdpr/v1/opendsr_requests + body: > + { "subject_request_id":"f4e5a271-f25e-4107-b681-123456789012", + "subject_request_type":"erasure", + "submitted_time":"2023-05-15T10:00:00Z", "platform": "android", + "subject_identities":[ { + "identity_type":"android_advertising_id", + "identity_value":"4412345-4963-9234-632156789012", + "identity_format":"raw" } ], "api_version":"0.1", + "property_id":"com.madeup.app", "status_callback_urls":[ + "https://examplecontroller.com/opengdpr_callbacks" ] } diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py new file mode 100644 index 00000000000..2ec3ffaa4d3 --- /dev/null +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -0,0 +1,69 @@ +from typing import Any, Dict, Generator + +import pydash +import pytest + +from tests.ops.integration_tests.saas.connector_runner import ( + ConnectorRunner, + generate_random_email, +) +from tests.ops.test_helpers.vault_client import get_secrets + +secrets = get_secrets("appsflyer") + + +@pytest.fixture(scope="session") +def appsflyer_secrets(saas_config) -> Dict[str, Any]: + return { + "domain": pydash.get(saas_config, "appsflyer.domain") + or secrets["domain"] + # add the rest of your secrets here + } + + +@pytest.fixture(scope="session") +def appsflyer_identity_email(saas_config) -> str: + return ( + pydash.get(saas_config, "appsflyer.identity_email") or secrets["identity_email"] + ) + + +@pytest.fixture +def appsflyer_erasure_identity_email() -> str: + return generate_random_email() + + +@pytest.fixture +def appsflyer_external_references() -> Dict[str, Any]: + return {} + + +@pytest.fixture +def appsflyer_erasure_external_references() -> Dict[str, Any]: + return {} + + +@pytest.fixture +def appsflyer_erasure_data( + appsflyer_erasure_identity_email: str, +) -> Generator: + # create the data needed for erasure tests here + yield {} + + +@pytest.fixture +def appsflyer_runner( + db, + cache, + appsflyer_secrets, + appsflyer_external_references, + appsflyer_erasure_external_references, +) -> ConnectorRunner: + return ConnectorRunner( + db, + cache, + "appsflyer", + appsflyer_secrets, + external_references=appsflyer_external_references, + erasure_external_references=appsflyer_erasure_external_references, + ) \ No newline at end of file diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py new file mode 100644 index 00000000000..337c4f1cb7e --- /dev/null +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -0,0 +1,51 @@ +import pytest + +from fides.api.models.policy import Policy +from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner + + +@pytest.mark.integration_saas +class TestAppsflyerConnector: + def test_connection(self, appsflyer_runner: ConnectorRunner): + appsflyer_runner.test_connection() + + async def test_access_request( + self, appsflyer_runner: ConnectorRunner, policy, appsflyer_identity_email: str + ): + access_results = await appsflyer_runner.access_request( + access_policy=policy, identities={"email": appsflyer_identity_email} + ) + + async def test_strict_erasure_request( + self, + appsflyer_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + appsflyer_erasure_identity_email: str, + appsflyer_erasure_data, + ): + ( + access_results, + erasure_results, + ) = await appsflyer_runner.strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": appsflyer_erasure_identity_email}, + ) + + async def test_non_strict_erasure_request( + self, + appsflyer_runner: ConnectorRunner, + policy: Policy, + erasure_policy_string_rewrite: Policy, + appsflyer_erasure_identity_email: str, + appsflyer_erasure_data, + ): + ( + access_results, + erasure_results, + ) = await appsflyer_runner.non_strict_erasure_request( + access_policy=policy, + erasure_policy=erasure_policy_string_rewrite, + identities={"email": appsflyer_erasure_identity_email}, + ) \ No newline at end of file From 50d70ddbd71fed1df801dfccf3226db0a2a60337 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 29 Nov 2023 12:26:41 -0500 Subject: [PATCH 02/41] staging --- tests/fixtures/saas/appsflyer_fixtures.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 2ec3ffaa4d3..16dfa30123f 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -15,9 +15,10 @@ @pytest.fixture(scope="session") def appsflyer_secrets(saas_config) -> Dict[str, Any]: return { - "domain": pydash.get(saas_config, "appsflyer.domain") - or secrets["domain"] + "domain": pydash.get(saas_config, "appsflyer.domain") or secrets["domain"], # add the rest of your secrets here + "email": pydash.get(saas_config, "appsflyer.email") or secrets["email"], + "token": pydash.get(saas_config, "appsflyer.token") or secrets["token"], } @@ -66,4 +67,4 @@ def appsflyer_runner( appsflyer_secrets, external_references=appsflyer_external_references, erasure_external_references=appsflyer_erasure_external_references, - ) \ No newline at end of file + ) From db0b5bbb6b2510e243fdd4d150dcb7fdd838fac0 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 30 Nov 2023 10:55:44 -0500 Subject: [PATCH 03/41] Starting appsflyer connector building --- data/saas/config/appsflyer_config.yml | 28 ++++++++----------- data/saas/dataset/appsflyer_dataset.yml | 7 +++++ data/saas/icon/appsflyer.svg | 22 +++++++++++++++ tests/fixtures/saas/appsflyer_fixtures.py | 3 ++ .../saas/test_appsflyer_task.py | 6 +++- 5 files changed, 49 insertions(+), 17 deletions(-) create mode 100644 data/saas/icon/appsflyer.svg diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index fd98b5dac4e..8abf4e3d447 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -1,14 +1,21 @@ saas_config: fides_key: - name: Appsflyer SaaS Config + name: Appsflyer type: appsflyer description: A sample schema representing the Appsflyer connector for Fides + user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/appsflyer version: 0.1.0 connector_params: - name: domain default_value: hq1.appsflyer.com + description: Your Appsflyer domain + label: Domain - name: test_req_id + - name: identity_value + description: The identity value that is returned for the end user email being submitted. + label: Identity Value + # identity value is one we need to get from the customer client_config: protocol: https @@ -21,29 +28,18 @@ saas_config: endpoints: - name: erasure requests: - read: - method: GET - path: /api/gdpr/v1/opendsr_requests - body: > - { "subject_request_id":"f4e5a271-f25e-4107-b681-123456789015", - "subject_request_type":"access", - "submitted_time":"2023-05-15T10:00:00Z", "platform": "android", - "subject_identities":[ { - "identity_type":"android_advertising_id", - "identity_value":"4412345-4963-9234-632156789015", - "identity_format":"raw" } ], "api_version":"0.1", - "property_id":"com.madeup.app", "status_callback_urls":[ - "https://examplecontroller.com/opengdpr_callbacks" ] } delete: method: POST path: /api/gdpr/v1/opendsr_requests body: > - { "subject_request_id":"f4e5a271-f25e-4107-b681-123456789012", + { "subject_request_id":"", "subject_request_type":"erasure", "submitted_time":"2023-05-15T10:00:00Z", "platform": "android", "subject_identities":[ { "identity_type":"android_advertising_id", - "identity_value":"4412345-4963-9234-632156789012", + "identity_value":"", "identity_format":"raw" } ], "api_version":"0.1", "property_id":"com.madeup.app", "status_callback_urls":[ "https://examplecontroller.com/opengdpr_callbacks" ] } + +### Marc you are not done yet. Remove this comment only when you're DONE with this config diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index e69de29bb2d..a87ef6bee27 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -0,0 +1,7 @@ +### Marc you are not done yet. Remove this comment only when you're DONE with this dataset +dataset: + - fides_key: + name: appsflyer + description: A sample dataset representing the Appsflyer connector for Fides + collections: + - name: user diff --git a/data/saas/icon/appsflyer.svg b/data/saas/icon/appsflyer.svg new file mode 100644 index 00000000000..8e7a5fc3a67 --- /dev/null +++ b/data/saas/icon/appsflyer.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 16dfa30123f..70d7cffbfb7 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -1,3 +1,6 @@ +### Marc you are not done yet. Remove this comment only when you're DONE with this fixtures +### Do you have any extraneous fixtures? + from typing import Any, Dict, Generator import pydash diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index 337c4f1cb7e..0f661282ef5 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -1,3 +1,7 @@ +### Marc you are not done yet. Remove this comment only when you're DONE with this test file! +### Do you have any extraneous tests? +### All tests passing? + import pytest from fides.api.models.policy import Policy @@ -48,4 +52,4 @@ async def test_non_strict_erasure_request( access_policy=policy, erasure_policy=erasure_policy_string_rewrite, identities={"email": appsflyer_erasure_identity_email}, - ) \ No newline at end of file + ) From c6f5f5911eca8e4e498fd491f39214116498f7da Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Mon, 4 Dec 2023 12:16:23 -0500 Subject: [PATCH 04/41] typeo --- data/saas/config/appsflyer_config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 8abf4e3d447..81f77c62f58 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -10,8 +10,6 @@ saas_config: - name: domain default_value: hq1.appsflyer.com description: Your Appsflyer domain - label: Domain - - name: test_req_id - name: identity_value description: The identity value that is returned for the end user email being submitted. label: Identity Value From 92d57a55960cedf5c253a784ce27dc32191cd307 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 5 Dec 2023 10:55:19 -0500 Subject: [PATCH 05/41] Test Connection Working. --- data/saas/config/appsflyer_config.yml | 4 ++++ data/saas/dataset/appsflyer_dataset.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 81f77c62f58..ef52fb27c7c 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -23,6 +23,10 @@ saas_config: configuration: token: + test_request: + method: GET + path: /api/gdpr/v1/stub/discovery + endpoints: - name: erasure requests: diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index a87ef6bee27..054591e901e 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -5,3 +5,4 @@ dataset: description: A sample dataset representing the Appsflyer connector for Fides collections: - name: user + fields: [] From 59aa9210abc4baff341cc6a2a6814c026f1a1ffd Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 5 Dec 2023 12:09:33 -0500 Subject: [PATCH 06/41] Testing --- .../saas/test_appsflyer_task.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index 0f661282ef5..6d662090e22 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -20,23 +20,6 @@ async def test_access_request( access_policy=policy, identities={"email": appsflyer_identity_email} ) - async def test_strict_erasure_request( - self, - appsflyer_runner: ConnectorRunner, - policy: Policy, - erasure_policy_string_rewrite: Policy, - appsflyer_erasure_identity_email: str, - appsflyer_erasure_data, - ): - ( - access_results, - erasure_results, - ) = await appsflyer_runner.strict_erasure_request( - access_policy=policy, - erasure_policy=erasure_policy_string_rewrite, - identities={"email": appsflyer_erasure_identity_email}, - ) - async def test_non_strict_erasure_request( self, appsflyer_runner: ConnectorRunner, From a3818a09aa9fcece8a0fee38a9d13e011da63bdd Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 6 Dec 2023 11:07:18 -0500 Subject: [PATCH 07/41] Tweaks --- data/saas/config/appsflyer_config.yml | 11 ++++++++--- tests/fixtures/saas/appsflyer_fixtures.py | 3 ++- .../ops/integration_tests/saas/test_appsflyer_task.py | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index ef52fb27c7c..5ea3cc0236d 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -12,7 +12,8 @@ saas_config: description: Your Appsflyer domain - name: identity_value description: The identity value that is returned for the end user email being submitted. - label: Identity Value + - name: property_id + description: String representing the mobile app to which this request is scoped. # identity value is one we need to get from the customer client_config: @@ -34,14 +35,18 @@ saas_config: method: POST path: /api/gdpr/v1/opendsr_requests body: > - { "subject_request_id":"", + { "subject_request_id":"", "subject_request_type":"erasure", "submitted_time":"2023-05-15T10:00:00Z", "platform": "android", "subject_identities":[ { - "identity_type":"android_advertising_id", + "identity_type":"appsflyer_id", "identity_value":"", "identity_format":"raw" } ], "api_version":"0.1", "property_id":"com.madeup.app", "status_callback_urls":[ "https://examplecontroller.com/opengdpr_callbacks" ] } +# appsflyer_id is hardcoded in this case as my understanding is that the identity_value that will be provided to us is of this type. +# should probably also set submitted time to something like 'now' + ### Marc you are not done yet. Remove this comment only when you're DONE with this config +### Remember to ensure that the descriptions are included and the user guide! diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 70d7cffbfb7..7f693f458ee 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -19,8 +19,9 @@ def appsflyer_secrets(saas_config) -> Dict[str, Any]: return { "domain": pydash.get(saas_config, "appsflyer.domain") or secrets["domain"], - # add the rest of your secrets here "email": pydash.get(saas_config, "appsflyer.email") or secrets["email"], + "identity_value": pydash.get(saas_config, "appsflyer.email") + or secrets["identity_value"], "token": pydash.get(saas_config, "appsflyer.token") or secrets["token"], } diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index 6d662090e22..2bc60ff1c75 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -36,3 +36,4 @@ async def test_non_strict_erasure_request( erasure_policy=erasure_policy_string_rewrite, identities={"email": appsflyer_erasure_identity_email}, ) + assert erasure_results == {"appsflyer_instance:user": 1} From 6d8fd04b2fc5f008537d00bf377d21c6335a19b3 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 7 Dec 2023 11:58:35 -0500 Subject: [PATCH 08/41] tweaks --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 5ea3cc0236d..c6706fc494e 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -37,7 +37,7 @@ saas_config: body: > { "subject_request_id":"", "subject_request_type":"erasure", - "submitted_time":"2023-05-15T10:00:00Z", "platform": "android", + "submitted_time":"", "platform": "android", "subject_identities":[ { "identity_type":"appsflyer_id", "identity_value":"", From cb3b880239f4fd27edeaa3f59b40fe2554a27535 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 12 Dec 2023 08:34:57 -0500 Subject: [PATCH 09/41] Improved test request endpoint --- data/saas/config/appsflyer_config.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index c6706fc494e..e5dee523b4a 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -12,8 +12,6 @@ saas_config: description: Your Appsflyer domain - name: identity_value description: The identity value that is returned for the end user email being submitted. - - name: property_id - description: String representing the mobile app to which this request is scoped. # identity value is one we need to get from the customer client_config: @@ -26,7 +24,7 @@ saas_config: test_request: method: GET - path: /api/gdpr/v1/stub/discovery + path: /api/mng/apps?capabilities=&limit=1 endpoints: - name: erasure From b61133892142ec71f33f169ca14d1d7dd6850d94 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 12 Dec 2023 12:00:29 -0500 Subject: [PATCH 10/41] Correction --- data/saas/config/appsflyer_config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index e5dee523b4a..c28f42412fc 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -9,7 +9,7 @@ saas_config: connector_params: - name: domain default_value: hq1.appsflyer.com - description: Your Appsflyer domain + description: The Appsflyer domain. Default hq1.appsflyer.com - name: identity_value description: The identity value that is returned for the end user email being submitted. # identity value is one we need to get from the customer @@ -33,7 +33,7 @@ saas_config: method: POST path: /api/gdpr/v1/opendsr_requests body: > - { "subject_request_id":"", + { "subject_request_id":"", "subject_request_type":"erasure", "submitted_time":"", "platform": "android", "subject_identities":[ { From f611e4e82d6b357f07d87fc4d9561e9652f7e3a8 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 12 Dec 2023 14:53:24 -0500 Subject: [PATCH 11/41] tweaks --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index c28f42412fc..9785177bad7 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -33,7 +33,7 @@ saas_config: method: POST path: /api/gdpr/v1/opendsr_requests body: > - { "subject_request_id":"", + { "subject_request_id":"", "subject_request_type":"erasure", "submitted_time":"", "platform": "android", "subject_identities":[ { From 0a4dd44d87df63e2f41ad70ed7d7d755d85beb91 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 13 Dec 2023 14:50:03 -0500 Subject: [PATCH 12/41] tweaks --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 9785177bad7..9140259a303 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -40,7 +40,7 @@ saas_config: "identity_type":"appsflyer_id", "identity_value":"", "identity_format":"raw" } ], "api_version":"0.1", - "property_id":"com.madeup.app", "status_callback_urls":[ + "property_id":" Date: Thu, 14 Dec 2023 14:35:17 -0500 Subject: [PATCH 13/41] pass2 --- data/saas/config/appsflyer_config.yml | 11 ++++++++++- .../appsflyer_request_overrides.py | 0 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 9140259a303..c9c07d3db44 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -27,6 +27,15 @@ saas_config: path: /api/mng/apps?capabilities=&limit=1 endpoints: + - name: user + requests: + read: + request_override: appsflyer_get_app_names + param_values: + - name: property_id + references: + - appsflyer_property_id + - name: erasure requests: delete: @@ -40,7 +49,7 @@ saas_config: "identity_type":"appsflyer_id", "identity_value":"", "identity_format":"raw" } ], "api_version":"0.1", - "property_id":"", "status_callback_urls":[ "https://examplecontroller.com/opengdpr_callbacks" ] } # appsflyer_id is hardcoded in this case as my understanding is that the identity_value that will be provided to us is of this type. diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py new file mode 100644 index 00000000000..e69de29bb2d From 4c5b2221284f9fbe558df5d12f3e0df4206f58a6 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 14 Dec 2023 14:39:18 -0500 Subject: [PATCH 14/41] Appsflyer Overrides Start --- .../appsflyer_request_overrides.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index e69de29bb2d..8becfcdeb9c 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -0,0 +1,32 @@ +from typing import Any, Dict, List + +from fides.api.graph.traversal import TraversalNode +from fides.api.models.policy import Policy +from fides.api.models.privacy_request import PrivacyRequest +from fides.api.service.connectors.saas.authenticated_client import AuthenticatedClient +from fides.api.service.saas_request.saas_request_override_factory import ( + SaaSRequestType, + register, +) +from fides.api.util.collection_util import Row + + +@register("appsflyer_get_app_names", [SaaSRequestType.READ]) +def appsflyer_get_app_names( + client: AuthenticatedClient, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + input_data: Dict[str, List[Any]], + secrets: Dict[str, Any], +) -> List[Row]: + """ + Gather the full list of applications set up, and retrieve the application name and + platform information for use with erasure endpoint. + """ + + appsflyer_app_ids = input_data.get("user_id", []) + results = [] + for statsig_user_id in statsig_user_ids: + results.append({"id": statsig_user_id}) + return results From b20bbcc54574134e883a4e5aa06a96405c4fb81b Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 14 Dec 2023 14:44:45 -0500 Subject: [PATCH 15/41] Override pass 1 done --- .../override_implementations/appsflyer_request_overrides.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index 8becfcdeb9c..15953d91dce 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -25,8 +25,8 @@ def appsflyer_get_app_names( platform information for use with erasure endpoint. """ - appsflyer_app_ids = input_data.get("user_id", []) + appsflyer_app_ids = input_data.get("id", []) results = [] - for statsig_user_id in statsig_user_ids: - results.append({"id": statsig_user_id}) + for appsflyer_app_id in appsflyer_app_ids: + results.append({"id": appsflyer_app_id}) return results From e6b052df5c117da08fa008eccf78f92807f79083 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 14 Dec 2023 14:45:28 -0500 Subject: [PATCH 16/41] Hardcode date --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index c9c07d3db44..f90fcc42ffe 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -44,7 +44,7 @@ saas_config: body: > { "subject_request_id":"", "subject_request_type":"erasure", - "submitted_time":"", "platform": "android", + "submitted_time":"2023-11-20", "platform": "android", "subject_identities":[ { "identity_type":"appsflyer_id", "identity_value":"", From 9885e994b7495010a7cf90c916af9f820deaadba Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 14 Dec 2023 14:45:50 -0500 Subject: [PATCH 17/41] Notes --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index f90fcc42ffe..b23792195ac 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -53,7 +53,7 @@ saas_config: "https://examplecontroller.com/opengdpr_callbacks" ] } # appsflyer_id is hardcoded in this case as my understanding is that the identity_value that will be provided to us is of this type. -# should probably also set submitted time to something like 'now' +# should probably also set submitted time to something like 'now' hardcoding for now ### Marc you are not done yet. Remove this comment only when you're DONE with this config ### Remember to ensure that the descriptions are included and the user guide! From 5fb0c95b0ef30312e924ffbf2e175f3450badf9e Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Thu, 14 Dec 2023 14:59:58 -0500 Subject: [PATCH 18/41] Config tweaks --- data/saas/config/appsflyer_config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index b23792195ac..3154069f5b0 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -14,6 +14,10 @@ saas_config: description: The identity value that is returned for the end user email being submitted. # identity value is one we need to get from the customer + external_references: + - name: appsflyer_property_id + description: Retrieved from the app list endpoint, a list of application names to use with erasure request + client_config: protocol: https host: hq1.appsflyer.com From 2c5e0da04de28e4ff5c7ffb4ebf299b9c33eb3e7 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Mon, 18 Dec 2023 14:43:44 -0500 Subject: [PATCH 19/41] External Ref making --- data/saas/config/appsflyer_config.yml | 2 +- .../appsflyer_request_overrides.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 3154069f5b0..f4e1e9bc329 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -46,7 +46,7 @@ saas_config: method: POST path: /api/gdpr/v1/opendsr_requests body: > - { "subject_request_id":"", + { "subject_request_id":"", "subject_request_type":"erasure", "submitted_time":"2023-11-20", "platform": "android", "subject_identities":[ { diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index 15953d91dce..004e68ed507 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -25,8 +25,17 @@ def appsflyer_get_app_names( platform information for use with erasure endpoint. """ - appsflyer_app_ids = input_data.get("id", []) - results = [] - for appsflyer_app_id in appsflyer_app_ids: - results.append({"id": appsflyer_app_id}) - return results + response = client.send( + SaaSRequestParams( + method=HTTPMethod.GET, + path="/api/mng/apps", + ) + ) + ## Getting a single value doesn't look too hard, but how to get more than one? + surveys = pydash.get(response.json(), "data") + ## need to loop to process all the app names (ids) we're about to go through + ## we need to submit a erasure request once with each app name to ensure we get + ## all the correct names for use. + ## can we do a for each based on the number of app names we get back and issue an + ## erasure request for each app name with all other params the same? + return [id[0]] if surveys else [] From 6521e1d7308b3106840106f4b9e7d40b34167b4e Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Mon, 18 Dec 2023 14:44:28 -0500 Subject: [PATCH 20/41] Initial changes --- .../override_implementations/appsflyer_request_overrides.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index 004e68ed507..c7613d4d324 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -32,10 +32,10 @@ def appsflyer_get_app_names( ) ) ## Getting a single value doesn't look too hard, but how to get more than one? - surveys = pydash.get(response.json(), "data") + app_names = pydash.get(response.json(), "data") ## need to loop to process all the app names (ids) we're about to go through ## we need to submit a erasure request once with each app name to ensure we get ## all the correct names for use. ## can we do a for each based on the number of app names we get back and issue an ## erasure request for each app name with all other params the same? - return [id[0]] if surveys else [] + return [id[0]] if app_names else [] From 5c1b3fd932b801f3f57e9bf3917ea62f5cdb36df Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Mon, 18 Dec 2023 16:31:58 -0500 Subject: [PATCH 21/41] Clean up description --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index f4e1e9bc329..3555f067d4f 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -11,7 +11,7 @@ saas_config: default_value: hq1.appsflyer.com description: The Appsflyer domain. Default hq1.appsflyer.com - name: identity_value - description: The identity value that is returned for the end user email being submitted. + description: The appsflyer_id for the user we are submitting the request for. # identity value is one we need to get from the customer external_references: From ccf2cd84f1962d689f7d540a665e6ba2248b6ff2 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 18 Dec 2023 14:37:43 -0800 Subject: [PATCH 22/41] Appsflyer WIP --- data/saas/config/appsflyer_config.yml | 78 ++++++++++++++----- .../appsflyer_request_overrides.py | 27 +++---- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 3555f067d4f..d0a273b85df 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -31,31 +31,69 @@ saas_config: path: /api/mng/apps?capabilities=&limit=1 endpoints: - - name: user + - name: apps requests: - read: - request_override: appsflyer_get_app_names - param_values: - - name: property_id - references: - - appsflyer_property_id - - - name: erasure + read: + method: GET + path: /api/mng/apps + param_values: + - name: email + identity: email + data_path: data + pagination: + strategy: link + configuration: + source: body + path: links.next + - name: user requests: + read: + request_override: appsflyer_user_read + param_values: + - name: app_id + value: + references: + - dataset: + field: apps.id + direction: from + - name: user_id + value: + references: + - appsflyer_user_id delete: method: POST path: /api/gdpr/v1/opendsr_requests - body: > - { "subject_request_id":"", - "subject_request_type":"erasure", - "submitted_time":"2023-11-20", "platform": "android", - "subject_identities":[ { - "identity_type":"appsflyer_id", - "identity_value":"", - "identity_format":"raw" } ], "api_version":"0.1", - "property_id":"", "status_callback_urls":[ - "https://examplecontroller.com/opengdpr_callbacks" ] } - + body: | + { + "subject_request_id": "", + "subject_request_type": "erasure", + "submitted_time": "2023-11-20", + "subject_identities": [ + { + "identity_type": "appsflyer_id", + "identity_value": "", + "identity_format": "raw" + } + ], + "api_version":"0.1", + "property_id": "", + "status_callback_urls":[ + "https://examplecontroller.com/opengdpr_callbacks" + ] + } + param_values: + - name: user_id + value: + references: + - dataset: + field: user.id + direction: from + - name: app_id + value: + references: + - dataset: + field: user.app_id + direction: from # appsflyer_id is hardcoded in this case as my understanding is that the identity_value that will be provided to us is of this type. # should probably also set submitted time to something like 'now' hardcoding for now diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index c7613d4d324..fb9979689ae 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -11,8 +11,8 @@ from fides.api.util.collection_util import Row -@register("appsflyer_get_app_names", [SaaSRequestType.READ]) -def appsflyer_get_app_names( +@register("appsflyer_user_read", [SaaSRequestType.READ]) +def appsflyer_user_read( client: AuthenticatedClient, node: TraversalNode, policy: Policy, @@ -25,17 +25,12 @@ def appsflyer_get_app_names( platform information for use with erasure endpoint. """ - response = client.send( - SaaSRequestParams( - method=HTTPMethod.GET, - path="/api/mng/apps", - ) - ) - ## Getting a single value doesn't look too hard, but how to get more than one? - app_names = pydash.get(response.json(), "data") - ## need to loop to process all the app names (ids) we're about to go through - ## we need to submit a erasure request once with each app name to ensure we get - ## all the correct names for use. - ## can we do a for each based on the number of app names we get back and issue an - ## erasure request for each app name with all other params the same? - return [id[0]] if app_names else [] + app_ids = input_data.get("app_id") + user_id = input_data.get("user_id") + + # join these together + users = [] + + # [{"id": "123", "app_id": "fides"}, ..., ..., ...] + + return users From e6eb7fcfd92d93fa2164e437c1a446ee43a2d137 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 10:13:19 -0500 Subject: [PATCH 23/41] Preliminary Changes --- data/saas/config/appsflyer_config.yml | 8 +++++--- .../appsflyer_request_overrides.py | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index d0a273b85df..34ac32bc7a2 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -11,11 +11,14 @@ saas_config: default_value: hq1.appsflyer.com description: The Appsflyer domain. Default hq1.appsflyer.com - name: identity_value - description: The appsflyer_id for the user we are submitting the request for. - # identity value is one we need to get from the customer + description: The location in your owned database where you store Appsflyer IDs + - name: API token + label: Appsflyer API Token + description: Appsflyer API Token (https://support.appsflyer.com/hc/en-us/articles/360004562377#retrieving-api-tokens) external_references: - name: appsflyer_property_id + label: Appsflyer App Name description: Retrieved from the app list endpoint, a list of application names to use with erasure request client_config: @@ -98,4 +101,3 @@ saas_config: # should probably also set submitted time to something like 'now' hardcoding for now ### Marc you are not done yet. Remove this comment only when you're DONE with this config -### Remember to ensure that the descriptions are included and the user guide! diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index fb9979689ae..b8798bb6d80 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -25,11 +25,13 @@ def appsflyer_user_read( platform information for use with erasure endpoint. """ - app_ids = input_data.get("app_id") user_id = input_data.get("user_id") + app_ids = input_data.get("app_id") - # join these together - users = [] + # join these together - such that we have a user_id paired with each app id + for app_id in app_ids: + users = [] + users.append({"user_id": user_id, "app_id": app_id}) # [{"id": "123", "app_id": "fides"}, ..., ..., ...] From 5515f2e506638133b96c5a104b430a794bc19621 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 10:36:52 -0500 Subject: [PATCH 24/41] Added ChangeLog message --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b4e9ee2090..62cd49bbd17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The types of changes are: - Added feature flag for separating system name and Compass vendor selector [#4437](https://github.com/ethyca/fides/pull/4437) - Fire GPP events per spec [#4433](https://github.com/ethyca/fides/pull/4433) - New override option `fides_tcf_gdpr_applies` for setting `gdprApplies` on the CMP API [#4453](https://github.com/ethyca/fides/pull/4453) +- Added Appsflyer Erasure Connector [#4512](https://github.com/ethyca/fides/pull/4512) ### Changed - Improved bulk vendor adding table UX [#4425](https://github.com/ethyca/fides/pull/4425) From 6f5ebe8463f060fb1435acdc71cf3dc74558f252 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 13:46:19 -0500 Subject: [PATCH 25/41] WIP dataset modification --- data/saas/config/appsflyer_config.yml | 41 ++++++++----------- data/saas/dataset/appsflyer_dataset.yml | 13 +++++- .../appsflyer_request_overrides.py | 11 +++-- tests/fixtures/saas/appsflyer_fixtures.py | 6 +-- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 34ac32bc7a2..da07fe0b648 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -17,9 +17,8 @@ saas_config: description: Appsflyer API Token (https://support.appsflyer.com/hc/en-us/articles/360004562377#retrieving-api-tokens) external_references: - - name: appsflyer_property_id - label: Appsflyer App Name - description: Retrieved from the app list endpoint, a list of application names to use with erasure request + - name: appsflyer_user_id + label: Appsflyer user ID field client_config: protocol: https @@ -54,15 +53,13 @@ saas_config: request_override: appsflyer_user_read param_values: - name: app_id - value: - references: - - dataset: - field: apps.id - direction: from + references: + - dataset: + field: apps.id + direction: from - name: user_id - value: - references: - - appsflyer_user_id + references: + - appsflyer_user_id delete: method: POST path: /api/gdpr/v1/opendsr_requests @@ -86,18 +83,12 @@ saas_config: } param_values: - name: user_id - value: - references: - - dataset: - field: user.id - direction: from + references: + - dataset: + field: user.id + direction: from - name: app_id - value: - references: - - dataset: - field: user.app_id - direction: from -# appsflyer_id is hardcoded in this case as my understanding is that the identity_value that will be provided to us is of this type. -# should probably also set submitted time to something like 'now' hardcoding for now - -### Marc you are not done yet. Remove this comment only when you're DONE with this config + references: + - dataset: + field: user.app_id + direction: from diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index 054591e901e..9607cdcec54 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -5,4 +5,15 @@ dataset: description: A sample dataset representing the Appsflyer connector for Fides collections: - name: user - fields: [] + fields: + - name: id + data_categories: [user.unique_id] + fidesops_meta: + primary_key: True + - name: app_id + data_categories: [app.unique_id] + fidesops_meta: + primary_key: True + + + [] # figure out how to do primary key announcement and document our new id and app_id diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index b8798bb6d80..9f3e993e7d9 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -25,13 +25,16 @@ def appsflyer_user_read( platform information for use with erasure endpoint. """ - user_id = input_data.get("user_id") - app_ids = input_data.get("app_id") + user_ids = input_data.get("user_id") # lists + app_ids = input_data.get("app_id") # lists + import pdb + pdb.set_trace() # join these together - such that we have a user_id paired with each app id + users = [] for app_id in app_ids: - users = [] - users.append({"user_id": user_id, "app_id": app_id}) + for user_id in user_ids: + users.append({"id": user_id, "app_id": app_id}) # [{"id": "123", "app_id": "fides"}, ..., ..., ...] diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 7f693f458ee..ab266fee852 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -40,19 +40,19 @@ def appsflyer_erasure_identity_email() -> str: @pytest.fixture def appsflyer_external_references() -> Dict[str, Any]: - return {} + return {"appsflyer_user_id": "4412345-4963-9234-632156789012"} @pytest.fixture def appsflyer_erasure_external_references() -> Dict[str, Any]: - return {} + return {"appsflyer_user_id": "4412345-4963-9234-632156780152"} @pytest.fixture def appsflyer_erasure_data( appsflyer_erasure_identity_email: str, ) -> Generator: - # create the data needed for erasure tests here + # create the data needed for erasure tests here - create user steps go here if we have them yield {} From 2edcf0355800a237be928ed3f0b3729c6b74daeb Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 15:16:32 -0500 Subject: [PATCH 26/41] WIP testing --- data/saas/config/appsflyer_config.yml | 5 +++-- data/saas/dataset/appsflyer_dataset.yml | 20 ++++++++----------- .../appsflyer_request_overrides.py | 2 +- tests/fixtures/saas/appsflyer_fixtures.py | 4 ++-- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index da07fe0b648..c75ee239fe6 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -65,9 +65,9 @@ saas_config: path: /api/gdpr/v1/opendsr_requests body: | { - "subject_request_id": "", + "subject_request_id": "69e58c53-88e2-4ca5-ba34-b6717723d1c6", "subject_request_type": "erasure", - "submitted_time": "2023-11-20", + "submitted_time": "2023-11-20T11:00:00Z", "subject_identities": [ { "identity_type": "appsflyer_id", @@ -92,3 +92,4 @@ saas_config: - dataset: field: user.app_id direction: from +# please use RFC3339 date string representation (e.g. 2018-05-01T11:00:00Z)"}}' diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index 9607cdcec54..48ae2e52cc5 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -5,15 +5,11 @@ dataset: description: A sample dataset representing the Appsflyer connector for Fides collections: - name: user - fields: - - name: id - data_categories: [user.unique_id] - fidesops_meta: - primary_key: True - - name: app_id - data_categories: [app.unique_id] - fidesops_meta: - primary_key: True - - - [] # figure out how to do primary key announcement and document our new id and app_id + fields: + - name: id + data_categories: [user.unique_id] + fidesops_meta: + primary_key: True + - name: app_id + data_categories: [system.operations] + fidesops_meta: diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index 9f3e993e7d9..96215102e49 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -29,7 +29,7 @@ def appsflyer_user_read( app_ids = input_data.get("app_id") # lists import pdb - pdb.set_trace() + # pdb.set_trace() # join these together - such that we have a user_id paired with each app id users = [] for app_id in app_ids: diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index ab266fee852..c39ac6ff1a5 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -40,12 +40,12 @@ def appsflyer_erasure_identity_email() -> str: @pytest.fixture def appsflyer_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "4412345-4963-9234-632156789012"} + return {"appsflyer_user_id": "518f723a-cfca-4916-ae1f-4a4cc0cf91a3"} @pytest.fixture def appsflyer_erasure_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "4412345-4963-9234-632156780152"} + return {"appsflyer_user_id": "0ccce139-d5a5-49d7-b53d-0cb9df417f33"} @pytest.fixture From 45c414a774a05d7eb01a35918dc57ccdf46882ec Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 15:19:35 -0500 Subject: [PATCH 27/41] WIP testing --- data/saas/config/appsflyer_config.yml | 2 +- tests/fixtures/saas/appsflyer_fixtures.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index c75ee239fe6..a8723079476 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -65,7 +65,7 @@ saas_config: path: /api/gdpr/v1/opendsr_requests body: | { - "subject_request_id": "69e58c53-88e2-4ca5-ba34-b6717723d1c6", + "subject_request_id": "c31e8ef1-2eff-44fe-a02e-f80851958fb3", "subject_request_type": "erasure", "submitted_time": "2023-11-20T11:00:00Z", "subject_identities": [ diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index c39ac6ff1a5..15a34e6d6bc 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -40,12 +40,12 @@ def appsflyer_erasure_identity_email() -> str: @pytest.fixture def appsflyer_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "518f723a-cfca-4916-ae1f-4a4cc0cf91a3"} + return {"appsflyer_user_id": "c31e8ef1-2eff-44fe-a02e-f80851958fb3"} @pytest.fixture def appsflyer_erasure_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "0ccce139-d5a5-49d7-b53d-0cb9df417f33"} + return {"appsflyer_user_id": "de33c37b-c45b-4d20-a05d-691bc765e354"} @pytest.fixture From 382f25f6c078291a06a95f646dced25179d5e07d Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 15:27:21 -0500 Subject: [PATCH 28/41] WIP testing --- data/saas/config/appsflyer_config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index a8723079476..cdf08db2ecc 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -92,4 +92,5 @@ saas_config: - dataset: field: user.app_id direction: from -# please use RFC3339 date string representation (e.g. 2018-05-01T11:00:00Z)"}}' +# Date is hardcoded in this case - dates need to be RFC3339 compliant - example 2018-05-01T11:00:00Z +# generates a string but it is not UUID v4 compatible so hardcoded at the moment From 1a7232e84f009376cf6cf98a67d5315617315579 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 19 Dec 2023 15:30:07 -0800 Subject: [PATCH 29/41] Adding new uuid and iso_8601_date placeholders --- data/saas/config/appsflyer_config.yml | 4 ++-- src/fides/api/service/connectors/saas_query_config.py | 9 +++++++++ src/fides/api/util/saas_util.py | 2 ++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index cdf08db2ecc..c0d208d24ac 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -65,9 +65,9 @@ saas_config: path: /api/gdpr/v1/opendsr_requests body: | { - "subject_request_id": "c31e8ef1-2eff-44fe-a02e-f80851958fb3", + "subject_request_id": "", "subject_request_type": "erasure", - "submitted_time": "2023-11-20T11:00:00Z", + "submitted_time": "", "subject_identities": [ { "identity_type": "appsflyer_id", diff --git a/src/fides/api/service/connectors/saas_query_config.py b/src/fides/api/service/connectors/saas_query_config.py index c73202c55a8..e7277c82b38 100644 --- a/src/fides/api/service/connectors/saas_query_config.py +++ b/src/fides/api/service/connectors/saas_query_config.py @@ -1,7 +1,9 @@ from __future__ import annotations +from datetime import datetime from itertools import product from typing import Any, Dict, List, Literal, Optional, TypeVar +from uuid import uuid4 import pydash from fideslang.models import FidesDatasetReference @@ -21,8 +23,10 @@ ALL_OBJECT_FIELDS, CUSTOM_PRIVACY_REQUEST_FIELDS, FIDESOPS_GROUPED_INPUTS, + ISO_8601_DATE, MASKED_OBJECT_FIELDS, PRIVACY_REQUEST_ID, + UUID, get_identity, unflatten_dict, ) @@ -307,6 +311,9 @@ def generate_query( 0 ] + param_values[UUID] = str(uuid4()) + param_values[ISO_8601_DATE] = datetime.now().date().isoformat() + # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = saas_util.map_param_values( self.action, self.collection_name, self.current_request, param_values # type: ignore @@ -403,6 +410,8 @@ def generate_update_param_values( # pylint: disable=R0914 param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields + param_values[UUID] = str(uuid4()) + param_values[ISO_8601_DATE] = datetime.now().date().isoformat() # remove any row values for fields marked as read-only, these will be omitted from all update maps for field_path, field in self.field_map().items(): diff --git a/src/fides/api/util/saas_util.py b/src/fides/api/util/saas_util.py index f651f743e9a..7efa0bb0f0d 100644 --- a/src/fides/api/util/saas_util.py +++ b/src/fides/api/util/saas_util.py @@ -25,6 +25,8 @@ MASKED_OBJECT_FIELDS = "masked_object_fields" ALL_OBJECT_FIELDS = "all_object_fields" CUSTOM_PRIVACY_REQUEST_FIELDS = "custom_privacy_request_fields" +UUID = "uuid" +ISO_8601_DATE = "iso_8601_date" def deny_unsafe_hosts(host: str) -> str: From 634937dade47421b78dad31f7f937406826d0b38 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Tue, 19 Dec 2023 15:38:34 -0500 Subject: [PATCH 30/41] WIP testing --- data/saas/config/appsflyer_config.yml | 2 ++ .../override_implementations/appsflyer_request_overrides.py | 2 +- tests/fixtures/saas/appsflyer_fixtures.py | 4 ++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index c0d208d24ac..e14a347e0ae 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -94,3 +94,5 @@ saas_config: direction: from # Date is hardcoded in this case - dates need to be RFC3339 compliant - example 2018-05-01T11:00:00Z # generates a string but it is not UUID v4 compatible so hardcoded at the moment +# In order for the non strict erasure to complete successfully we must have a unique UUID v4 string for the +# subject request id as well as a different unique UUID v4 for the appsflyer id diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index 96215102e49..25c03f8c7b8 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -27,7 +27,7 @@ def appsflyer_user_read( user_ids = input_data.get("user_id") # lists app_ids = input_data.get("app_id") # lists - import pdb + # import pdb # pdb.set_trace() # join these together - such that we have a user_id paired with each app id diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 15a34e6d6bc..2cba81fee33 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -40,12 +40,12 @@ def appsflyer_erasure_identity_email() -> str: @pytest.fixture def appsflyer_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "c31e8ef1-2eff-44fe-a02e-f80851958fb3"} + return {"appsflyer_user_id": "ef6c1b95-dbd4-4088-93be-ebfdff100d95"} @pytest.fixture def appsflyer_erasure_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "de33c37b-c45b-4d20-a05d-691bc765e354"} + return {"appsflyer_user_id": "37dda6ed-18df-4b6c-adfe-534c2600f959"} @pytest.fixture From 61ece47c01c98e2f51af253a941110d2c7ecd978 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 20 Dec 2023 09:03:21 -0500 Subject: [PATCH 31/41] Removing some comments --- data/saas/config/appsflyer_config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index e14a347e0ae..9b972153f7a 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -92,7 +92,3 @@ saas_config: - dataset: field: user.app_id direction: from -# Date is hardcoded in this case - dates need to be RFC3339 compliant - example 2018-05-01T11:00:00Z -# generates a string but it is not UUID v4 compatible so hardcoded at the moment -# In order for the non strict erasure to complete successfully we must have a unique UUID v4 string for the -# subject request id as well as a different unique UUID v4 for the appsflyer id From b8132966cb6e1c2ec87de60e1da3ec6c59bd1b64 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 20 Dec 2023 09:43:15 -0500 Subject: [PATCH 32/41] Tweaks to get variables in desired format --- data/saas/dataset/appsflyer_dataset.yml | 1 + src/fides/api/service/connectors/saas_query_config.py | 2 +- tests/fixtures/saas/appsflyer_fixtures.py | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index 48ae2e52cc5..b259defedba 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -13,3 +13,4 @@ dataset: - name: app_id data_categories: [system.operations] fidesops_meta: + # primary_key: True diff --git a/src/fides/api/service/connectors/saas_query_config.py b/src/fides/api/service/connectors/saas_query_config.py index e7277c82b38..4a58e6f39b0 100644 --- a/src/fides/api/service/connectors/saas_query_config.py +++ b/src/fides/api/service/connectors/saas_query_config.py @@ -411,7 +411,7 @@ def generate_update_param_values( # pylint: disable=R0914 param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields param_values[UUID] = str(uuid4()) - param_values[ISO_8601_DATE] = datetime.now().date().isoformat() + param_values[ISO_8601_DATE] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ") # remove any row values for fields marked as read-only, these will be omitted from all update maps for field_path, field in self.field_map().items(): diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 2cba81fee33..b5b4b59537b 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -40,12 +40,12 @@ def appsflyer_erasure_identity_email() -> str: @pytest.fixture def appsflyer_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "ef6c1b95-dbd4-4088-93be-ebfdff100d95"} + return {"appsflyer_user_id": ""} @pytest.fixture def appsflyer_erasure_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": "37dda6ed-18df-4b6c-adfe-534c2600f959"} + return {"appsflyer_user_id": ""} @pytest.fixture From 08b32e201d1c35aa2559f96bcc42984c32c921e3 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 20 Dec 2023 12:19:35 -0500 Subject: [PATCH 33/41] Tests passing --- tests/fixtures/saas/appsflyer_fixtures.py | 5 +++-- tests/ops/integration_tests/saas/test_appsflyer_task.py | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index b5b4b59537b..2a23c8dd02e 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -2,6 +2,7 @@ ### Do you have any extraneous fixtures? from typing import Any, Dict, Generator +import uuid import pydash import pytest @@ -40,12 +41,12 @@ def appsflyer_erasure_identity_email() -> str: @pytest.fixture def appsflyer_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": ""} + return {"appsflyer_user_id": uuid.uuid4()} @pytest.fixture def appsflyer_erasure_external_references() -> Dict[str, Any]: - return {"appsflyer_user_id": ""} + return {"appsflyer_user_id": uuid.uuid4()} @pytest.fixture diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index 2bc60ff1c75..a45042edc07 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -3,6 +3,7 @@ ### All tests passing? import pytest +import pdb from fides.api.models.policy import Policy from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @@ -36,4 +37,9 @@ async def test_non_strict_erasure_request( erasure_policy=erasure_policy_string_rewrite, identities={"email": appsflyer_erasure_identity_email}, ) - assert erasure_results == {"appsflyer_instance:user": 1} + # pdb.set_trace() + assert erasure_results == { + "appsflyer_external_dataset:appsflyer_external_collection": 0, + "appsflyer_instance:apps": 0, + "appsflyer_instance:user": 10, + } From 57452eb8a13ab7d01d7b7a0e40a5e12e88c393de Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 20 Dec 2023 12:24:49 -0500 Subject: [PATCH 34/41] Fixed date to be datetime --- data/saas/config/appsflyer_config.yml | 2 +- src/fides/api/service/connectors/saas_query_config.py | 6 +++--- src/fides/api/util/saas_util.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index 9b972153f7a..cfe06359899 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -67,7 +67,7 @@ saas_config: { "subject_request_id": "", "subject_request_type": "erasure", - "submitted_time": "", + "submitted_time": "", "subject_identities": [ { "identity_type": "appsflyer_id", diff --git a/src/fides/api/service/connectors/saas_query_config.py b/src/fides/api/service/connectors/saas_query_config.py index 4a58e6f39b0..c29b4705da0 100644 --- a/src/fides/api/service/connectors/saas_query_config.py +++ b/src/fides/api/service/connectors/saas_query_config.py @@ -23,7 +23,7 @@ ALL_OBJECT_FIELDS, CUSTOM_PRIVACY_REQUEST_FIELDS, FIDESOPS_GROUPED_INPUTS, - ISO_8601_DATE, + ISO_8601_DATETIME, MASKED_OBJECT_FIELDS, PRIVACY_REQUEST_ID, UUID, @@ -312,7 +312,7 @@ def generate_query( ] param_values[UUID] = str(uuid4()) - param_values[ISO_8601_DATE] = datetime.now().date().isoformat() + param_values[ISO_8601_DATETIME] = datetime.now().date().isoformat() # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = saas_util.map_param_values( @@ -411,7 +411,7 @@ def generate_update_param_values( # pylint: disable=R0914 param_values[CUSTOM_PRIVACY_REQUEST_FIELDS] = custom_privacy_request_fields param_values[UUID] = str(uuid4()) - param_values[ISO_8601_DATE] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ") + param_values[ISO_8601_DATETIME] = datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ") # remove any row values for fields marked as read-only, these will be omitted from all update maps for field_path, field in self.field_map().items(): diff --git a/src/fides/api/util/saas_util.py b/src/fides/api/util/saas_util.py index 7efa0bb0f0d..01afdec840a 100644 --- a/src/fides/api/util/saas_util.py +++ b/src/fides/api/util/saas_util.py @@ -26,7 +26,7 @@ ALL_OBJECT_FIELDS = "all_object_fields" CUSTOM_PRIVACY_REQUEST_FIELDS = "custom_privacy_request_fields" UUID = "uuid" -ISO_8601_DATE = "iso_8601_date" +ISO_8601_DATETIME = "iso_8601_datetime" def deny_unsafe_hosts(host: str) -> str: From f14e54aa06eb0f9bc2120822d75ca65ac3fb20d2 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Wed, 3 Jan 2024 09:52:39 -0500 Subject: [PATCH 35/41] Clean up --- data/saas/dataset/appsflyer_dataset.yml | 2 -- tests/fixtures/saas/appsflyer_fixtures.py | 3 --- tests/ops/integration_tests/saas/test_appsflyer_task.py | 9 +++------ 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index b259defedba..de77b98cd56 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -1,4 +1,3 @@ -### Marc you are not done yet. Remove this comment only when you're DONE with this dataset dataset: - fides_key: name: appsflyer @@ -13,4 +12,3 @@ dataset: - name: app_id data_categories: [system.operations] fidesops_meta: - # primary_key: True diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 2a23c8dd02e..9239d9e0fc9 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -1,6 +1,3 @@ -### Marc you are not done yet. Remove this comment only when you're DONE with this fixtures -### Do you have any extraneous fixtures? - from typing import Any, Dict, Generator import uuid diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index a45042edc07..38c3fec91af 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -1,9 +1,6 @@ -### Marc you are not done yet. Remove this comment only when you're DONE with this test file! -### Do you have any extraneous tests? -### All tests passing? - import pytest -import pdb + +# import pdb -- used for trouble shooting from fides.api.models.policy import Policy from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @@ -37,7 +34,7 @@ async def test_non_strict_erasure_request( erasure_policy=erasure_policy_string_rewrite, identities={"email": appsflyer_erasure_identity_email}, ) - # pdb.set_trace() + # pdb.set_trace() -- used for trouble shooting assert erasure_results == { "appsflyer_external_dataset:appsflyer_external_collection": 0, "appsflyer_instance:apps": 0, From c9279ba5daa636fbcd73daa6b2c1351842fefdc4 Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Fri, 5 Jan 2024 09:02:17 -0500 Subject: [PATCH 36/41] Corrections to Appsflyer casing --- data/saas/config/appsflyer_config.yml | 12 ++++++------ data/saas/dataset/appsflyer_dataset.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index cfe06359899..eb2703482f4 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -2,23 +2,23 @@ saas_config: fides_key: name: Appsflyer type: appsflyer - description: A sample schema representing the Appsflyer connector for Fides + description: A sample schema representing the AppsFlyer connector for Fides user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/appsflyer version: 0.1.0 connector_params: - name: domain default_value: hq1.appsflyer.com - description: The Appsflyer domain. Default hq1.appsflyer.com + description: The AppsFlyer domain. Default hq1.appsflyer.com - name: identity_value - description: The location in your owned database where you store Appsflyer IDs + description: The location in your owned database where you store AppsFlyer IDs - name: API token - label: Appsflyer API Token - description: Appsflyer API Token (https://support.appsflyer.com/hc/en-us/articles/360004562377#retrieving-api-tokens) + label: AppsFlyer API Token + description: AppsFlyer API Token (https://support.appsflyer.com/hc/en-us/articles/360004562377#retrieving-api-tokens) external_references: - name: appsflyer_user_id - label: Appsflyer user ID field + label: AppsFlyer user ID field client_config: protocol: https diff --git a/data/saas/dataset/appsflyer_dataset.yml b/data/saas/dataset/appsflyer_dataset.yml index de77b98cd56..3ad0afa8b9f 100644 --- a/data/saas/dataset/appsflyer_dataset.yml +++ b/data/saas/dataset/appsflyer_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: appsflyer - description: A sample dataset representing the Appsflyer connector for Fides + description: A sample dataset representing the AppsFlyer connector for Fides collections: - name: user fields: From 72d770599f4f9f80301707062ced0d6f78c2735b Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Fri, 5 Jan 2024 09:05:04 -0500 Subject: [PATCH 37/41] Removing commented out comments --- .../appsflyer_request_overrides.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py index 25c03f8c7b8..423c60a2b61 100644 --- a/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py +++ b/src/fides/api/service/saas_request/override_implementations/appsflyer_request_overrides.py @@ -25,17 +25,12 @@ def appsflyer_user_read( platform information for use with erasure endpoint. """ - user_ids = input_data.get("user_id") # lists - app_ids = input_data.get("app_id") # lists - # import pdb + user_ids = input_data.get("user_id") + app_ids = input_data.get("app_id") - # pdb.set_trace() - # join these together - such that we have a user_id paired with each app id users = [] for app_id in app_ids: for user_id in user_ids: users.append({"id": user_id, "app_id": app_id}) - # [{"id": "123", "app_id": "fides"}, ..., ..., ...] - return users From e845f386fb745118dfe1454d382a5ce1c6f551cd Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Fri, 5 Jan 2024 12:27:27 -0500 Subject: [PATCH 38/41] Added Assert --- tests/ops/integration_tests/saas/test_appsflyer_task.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index 38c3fec91af..6bfb9a01026 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -1,7 +1,5 @@ import pytest -# import pdb -- used for trouble shooting - from fides.api.models.policy import Policy from tests.ops.integration_tests.saas.connector_runner import ConnectorRunner @@ -18,6 +16,8 @@ async def test_access_request( access_policy=policy, identities={"email": appsflyer_identity_email} ) + assert len(access_results["appsflyer_instance:user"]) == 10 + async def test_non_strict_erasure_request( self, appsflyer_runner: ConnectorRunner, @@ -34,7 +34,6 @@ async def test_non_strict_erasure_request( erasure_policy=erasure_policy_string_rewrite, identities={"email": appsflyer_erasure_identity_email}, ) - # pdb.set_trace() -- used for trouble shooting assert erasure_results == { "appsflyer_external_dataset:appsflyer_external_collection": 0, "appsflyer_instance:apps": 0, From af9e3538f55384e044e7d4d8180a3f3843376faf Mon Sep 17 00:00:00 2001 From: Marc Guimond Date: Fri, 5 Jan 2024 13:08:36 -0500 Subject: [PATCH 39/41] Fixed casing for AppsFlyer in changelog.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83dc5951de3..a31a1b812a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,7 +82,7 @@ The types of changes are: - Added feature flag for separating system name and Compass vendor selector [#4437](https://github.com/ethyca/fides/pull/4437) - Fire GPP events per spec [#4433](https://github.com/ethyca/fides/pull/4433) - New override option `fides_tcf_gdpr_applies` for setting `gdprApplies` on the CMP API [#4453](https://github.com/ethyca/fides/pull/4453) -- Added Appsflyer Erasure Connector [#4512](https://github.com/ethyca/fides/pull/4512) +- Added AppsFlyer Erasure Connector [#4512](https://github.com/ethyca/fides/pull/4512) ### Changed - Improved bulk vendor adding table UX [#4425](https://github.com/ethyca/fides/pull/4425) From 6f37d62bafa3abd42892869d6d5b475110b29ef7 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 8 Jan 2024 20:50:18 -0800 Subject: [PATCH 40/41] Misc fixes --- CHANGELOG.md | 4 +++- data/saas/config/appsflyer_config.yml | 11 +++++------ tests/fixtures/saas/appsflyer_fixtures.py | 16 +++------------- .../saas/test_appsflyer_task.py | 5 ++--- 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a31a1b812a0..a91f50a9172 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fides/compare/2.27.0...main) +### Added +- Erasure support for AppsFlyer [#4512](https://github.com/ethyca/fides/pull/4512) + ## [2.27.0](https://github.com/ethyca/fides/compare/2.26.0...2.27.0) ### Added @@ -82,7 +85,6 @@ The types of changes are: - Added feature flag for separating system name and Compass vendor selector [#4437](https://github.com/ethyca/fides/pull/4437) - Fire GPP events per spec [#4433](https://github.com/ethyca/fides/pull/4433) - New override option `fides_tcf_gdpr_applies` for setting `gdprApplies` on the CMP API [#4453](https://github.com/ethyca/fides/pull/4453) -- Added AppsFlyer Erasure Connector [#4512](https://github.com/ethyca/fides/pull/4512) ### Changed - Improved bulk vendor adding table UX [#4425](https://github.com/ethyca/fides/pull/4425) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index eb2703482f4..ef397846e6e 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Appsflyer + name: AppsFlyer type: appsflyer description: A sample schema representing the AppsFlyer connector for Fides user_guide: https://docs.ethyca.com/user-guides/integrations/saas-integrations/appsflyer @@ -10,15 +10,14 @@ saas_config: - name: domain default_value: hq1.appsflyer.com description: The AppsFlyer domain. Default hq1.appsflyer.com - - name: identity_value - description: The location in your owned database where you store AppsFlyer IDs - - name: API token - label: AppsFlyer API Token + - name: api_token + label: API token description: AppsFlyer API Token (https://support.appsflyer.com/hc/en-us/articles/360004562377#retrieving-api-tokens) external_references: - name: appsflyer_user_id label: AppsFlyer user ID field + description: The location in your owned database where you store AppsFlyer IDs client_config: protocol: https @@ -26,7 +25,7 @@ saas_config: authentication: strategy: bearer configuration: - token: + token: test_request: method: GET diff --git a/tests/fixtures/saas/appsflyer_fixtures.py b/tests/fixtures/saas/appsflyer_fixtures.py index 9239d9e0fc9..9a027de8aff 100644 --- a/tests/fixtures/saas/appsflyer_fixtures.py +++ b/tests/fixtures/saas/appsflyer_fixtures.py @@ -1,5 +1,5 @@ -from typing import Any, Dict, Generator import uuid +from typing import Any, Dict import pydash import pytest @@ -17,10 +17,8 @@ def appsflyer_secrets(saas_config) -> Dict[str, Any]: return { "domain": pydash.get(saas_config, "appsflyer.domain") or secrets["domain"], - "email": pydash.get(saas_config, "appsflyer.email") or secrets["email"], - "identity_value": pydash.get(saas_config, "appsflyer.email") - or secrets["identity_value"], - "token": pydash.get(saas_config, "appsflyer.token") or secrets["token"], + "api_token": pydash.get(saas_config, "appsflyer.api_token") + or secrets["api_token"], } @@ -46,14 +44,6 @@ def appsflyer_erasure_external_references() -> Dict[str, Any]: return {"appsflyer_user_id": uuid.uuid4()} -@pytest.fixture -def appsflyer_erasure_data( - appsflyer_erasure_identity_email: str, -) -> Generator: - # create the data needed for erasure tests here - create user steps go here if we have them - yield {} - - @pytest.fixture def appsflyer_runner( db, diff --git a/tests/ops/integration_tests/saas/test_appsflyer_task.py b/tests/ops/integration_tests/saas/test_appsflyer_task.py index 6bfb9a01026..e5db12c0db4 100644 --- a/tests/ops/integration_tests/saas/test_appsflyer_task.py +++ b/tests/ops/integration_tests/saas/test_appsflyer_task.py @@ -5,7 +5,7 @@ @pytest.mark.integration_saas -class TestAppsflyerConnector: +class TestAppsFlyerConnector: def test_connection(self, appsflyer_runner: ConnectorRunner): appsflyer_runner.test_connection() @@ -24,10 +24,9 @@ async def test_non_strict_erasure_request( policy: Policy, erasure_policy_string_rewrite: Policy, appsflyer_erasure_identity_email: str, - appsflyer_erasure_data, ): ( - access_results, + _, erasure_results, ) = await appsflyer_runner.non_strict_erasure_request( access_policy=policy, From 4b8f7c9f2b626b1374a01054dc09e2fef0e8b4a0 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 8 Jan 2024 20:54:26 -0800 Subject: [PATCH 41/41] Updating client config with dynamic host --- data/saas/config/appsflyer_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/saas/config/appsflyer_config.yml b/data/saas/config/appsflyer_config.yml index ef397846e6e..385acd0a1d4 100644 --- a/data/saas/config/appsflyer_config.yml +++ b/data/saas/config/appsflyer_config.yml @@ -21,7 +21,7 @@ saas_config: client_config: protocol: https - host: hq1.appsflyer.com + host: authentication: strategy: bearer configuration: