From 04fc4f8d0318f17d1785fa4e2d16f80897b6013d Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Wed, 11 Oct 2023 21:22:58 -0400 Subject: [PATCH] fides: add `System` model support for new tcf fields (#4228) Co-authored-by: Dawn Pattison --- .fides/db_dataset.yml | 18 +++ CHANGELOG.md | 1 + .../81886da90395_add_legal_basis_dimension.py | 2 +- ...a218831820_additional_tcf_fields_system.py | 89 +++++++++++ src/fides/api/models/sql_models.py | 10 ++ src/fides/api/schemas/tcf.py | 7 +- .../api/util/tcf/tcf_experience_contents.py | 60 +++++++ tests/ctl/core/test_api.py | 14 ++ .../ops/util/test_tcf_experience_contents.py | 151 +++++++++++++++++- 9 files changed, 344 insertions(+), 8 deletions(-) create mode 100644 src/fides/api/alembic/migrations/versions/c5a218831820_additional_tcf_fields_system.py diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 17a37eb9624..56c311d4638 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -1142,6 +1142,21 @@ dataset: - name: data_security_practices data_categories: - system.operations + - name: cookie_max_age_seconds + data_categories: + - system.operations + - name: cookie_refresh + data_categories: + - system.operations + - name: uses_cookies + data_categories: + - system.operations + - name: uses_non_cookie_access + data_categories: + - system.operations + - name: legitimate_interest_disclosure_url + data_categories: + - system.operations - name: user_id data_categories: - user.unique_id @@ -1553,6 +1568,9 @@ dataset: - name: legal_basis_for_processing data_categories: - system.operations + - name: flexible_legal_basis_for_processing + data_categories: + - system.operations - name: impact_assessment_location data_categories: - system.operations diff --git a/CHANGELOG.md b/CHANGELOG.md index f5b5e529c79..2a75ec008ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The types of changes are: - Added an option to link to vendor tab from an experience config description [#4191](https://github.com/ethyca/fides/pull/4191) - Added two toggles for vendors in the TCF overlay, one for Consent, and one for Legitimate Interest [#4189](https://github.com/ethyca/fides/pull/4189) - Added two toggles for purposes in the TCF overlay, one for Consent, and one for Legitimate Interest [#4234](https://github.com/ethyca/fides/pull/4234) +- Added support for new TCF-related fields on `System` and `PrivacyDeclaration` models [#4228](https://github.com/ethyca/fides/pull/4228) ### Changed diff --git a/src/fides/api/alembic/migrations/versions/81886da90395_add_legal_basis_dimension.py b/src/fides/api/alembic/migrations/versions/81886da90395_add_legal_basis_dimension.py index 177a55fb43f..5dc8c1a129c 100644 --- a/src/fides/api/alembic/migrations/versions/81886da90395_add_legal_basis_dimension.py +++ b/src/fides/api/alembic/migrations/versions/81886da90395_add_legal_basis_dimension.py @@ -4,7 +4,7 @@ need to instead save preferences against a purpose/vendor AND a legal basis. Revision ID: 81886da90395 -Revises: 4cb3b5af4160 +Revises: 9b98aba5bba8 Create Date: 2023-09-30 17:39:46.251444 """ diff --git a/src/fides/api/alembic/migrations/versions/c5a218831820_additional_tcf_fields_system.py b/src/fides/api/alembic/migrations/versions/c5a218831820_additional_tcf_fields_system.py new file mode 100644 index 00000000000..23a805c5fe1 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/c5a218831820_additional_tcf_fields_system.py @@ -0,0 +1,89 @@ +"""additional tcf fields system + +Revision ID: c5a218831820 +Revises: 81226042d7d4 +Create Date: 2023-10-05 15:40:09.294013 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c5a218831820" +down_revision = "81226042d7d4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "ctl_systems", + sa.Column("cookie_max_age_seconds", sa.Integer(), nullable=True), + ) + op.add_column( + "ctl_systems", + sa.Column( + "uses_cookies", + sa.Boolean(), + nullable=False, + server_default="f", + ), + ) + op.add_column( + "ctl_systems", + sa.Column( + "cookie_refresh", + sa.Boolean(), + nullable=False, + server_default="f", + ), + ) + op.add_column( + "ctl_systems", + sa.Column( + "uses_non_cookie_access", + sa.Boolean(), + nullable=False, + server_default="f", + ), + ) + op.add_column( + "ctl_systems", + sa.Column("legitimate_interest_disclosure_url", sa.String(), nullable=True), + ) + + op.add_column( + "privacydeclaration", + sa.Column( + "flexible_legal_basis_for_processing", + sa.Boolean(), + nullable=True, + ), + ) + + +def downgrade(): + op.drop_column( + "ctl_systems", + "cookie_max_age_seconds", + ) + op.drop_column( + "ctl_systems", + "uses_cookies", + ) + op.drop_column( + "ctl_systems", + "cookie_refresh", + ) + op.drop_column( + "ctl_systems", + "uses_non_cookie_access", + ) + op.drop_column( + "ctl_systems", + "legitimate_interest_disclosure_url", + ) + op.drop_column( + "privacydeclaration", + "flexible_legal_basis_for_processing", + ) diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 8e079ca38d5..856b3b7d6bf 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -397,6 +397,15 @@ class System(Base, FidesBase): dpo = Column(String) joint_controller_info = Column(String) data_security_practices = Column(String) + cookie_max_age_seconds = Column(Integer) + uses_cookies = Column(BOOLEAN(), default=False, server_default="f", nullable=False) + cookie_refresh = Column( + BOOLEAN(), default=False, server_default="f", nullable=False + ) + uses_non_cookie_access = Column( + BOOLEAN(), default=False, server_default="f", nullable=False + ) + legitimate_interest_disclosure_url = Column(String) privacy_declarations = relationship( "PrivacyDeclaration", @@ -465,6 +474,7 @@ class PrivacyDeclaration(Base): features = Column(ARRAY(String), server_default="{}", nullable=False) legal_basis_for_processing = Column(String) + flexible_legal_basis_for_processing = Column(BOOLEAN()) impact_assessment_location = Column(String) retention_period = Column(String) processes_special_category_data = Column( diff --git a/src/fides/api/schemas/tcf.py b/src/fides/api/schemas/tcf.py index 617427c3901..e16ee3932b4 100644 --- a/src/fides/api/schemas/tcf.py +++ b/src/fides/api/schemas/tcf.py @@ -7,7 +7,7 @@ MAPPED_SPECIAL_PURPOSES, ) from fideslang.gvl.models import Feature, MappedPurpose -from pydantic import Field, root_validator, validator +from pydantic import AnyUrl, Field, root_validator, validator from fides.api.models.privacy_notice import UserConsentPreference from fides.api.schemas.base_class import FidesSchema @@ -110,6 +110,11 @@ class TCFVendorRelationships(CommonVendorFields): special_purposes: List[EmbeddedLineItem] = [] features: List[EmbeddedLineItem] = [] special_features: List[EmbeddedLineItem] = [] + cookie_max_age_seconds: Optional[int] + uses_cookies: Optional[bool] + cookie_refresh: Optional[bool] + uses_non_cookie_access: Optional[bool] + legitimate_interest_disclosure_url: Optional[AnyUrl] class TCFFeatureRecord(NonVendorSection, Feature): diff --git a/src/fides/api/util/tcf/tcf_experience_contents.py b/src/fides/api/util/tcf/tcf_experience_contents.py index 81c6a26024f..780f1f50219 100644 --- a/src/fides/api/util/tcf/tcf_experience_contents.py +++ b/src/fides/api/util/tcf/tcf_experience_contents.py @@ -184,6 +184,13 @@ def get_matching_privacy_declarations(db: Session) -> Query: System.fides_key.label("system_fides_key"), System.name.label("system_name"), System.description.label("system_description"), + System.cookie_max_age_seconds.label("system_cookie_max_age_seconds"), + System.uses_cookies.label("system_uses_cookies"), + System.cookie_refresh.label("system_cookie_refresh"), + System.uses_non_cookie_access.label("system_uses_non_cookie_access"), + System.legitimate_interest_disclosure_url.label( + "system_legitimate_interest_disclosure_url" + ), System.vendor_id, PrivacyDeclaration.data_use, PrivacyDeclaration.legal_basis_for_processing, @@ -418,6 +425,50 @@ def build_purpose_or_feature_section_and_update_vendor_map( return non_vendor_record_map, vendor_map +def populate_vendor_relationships_basic_attributes( + vendor_map: Dict[str, TCFVendorRelationships], + matching_privacy_declarations: Query, +) -> Dict[str, TCFVendorRelationships]: + """Populates TCFVendorRelationships records for all vendors that we're displaying in the overlay. + Ensures that these TCFVendorRelationships records have the "basic" TCF attributes populated. + """ + for privacy_declaration_row in matching_privacy_declarations: + vendor_id, system_identifier = get_system_identifiers(privacy_declaration_row) + + # Get the existing TCFVendorRelationships record or create a new one. + # Add to the vendor map if it wasn't added in a previous section. + vendor_relationship_record = vendor_map.get(system_identifier) + if not vendor_relationship_record: + vendor_relationship_record = TCFVendorRelationships( + id=system_identifier, # Identify system by vendor id if it exists, otherwise use system id. + name=privacy_declaration_row.system_name, + description=privacy_declaration_row.system_description, + has_vendor_id=bool( + vendor_id # This will let us separate data between systems and vendors later + ), + ) + vendor_map[system_identifier] = vendor_relationship_record + + # Now add basic attributes to the VendorRelationships record + vendor_relationship_record.cookie_max_age_seconds = ( + privacy_declaration_row.system_cookie_max_age_seconds + ) + vendor_relationship_record.uses_cookies = ( + privacy_declaration_row.system_uses_cookies + ) + vendor_relationship_record.cookie_refresh = ( + privacy_declaration_row.system_cookie_refresh + ) + vendor_relationship_record.uses_non_cookie_access = ( + privacy_declaration_row.system_uses_non_cookie_access + ) + vendor_relationship_record.legitimate_interest_disclosure_url = ( + privacy_declaration_row.system_legitimate_interest_disclosure_url + ) + + return vendor_map + + def get_tcf_contents( db: Session, ) -> TCFExperienceContents: @@ -511,6 +562,15 @@ def get_tcf_contents( matching_privacy_declaration_query=matching_privacy_declarations, ) + # Finally, add missing TCFVendorRelationships records for vendors that weren't already added + # via the special_features, features, and special_purposes section. Every vendor in the overlay + # should show up in this section. Add the basic attributes to the vendor here to avoid duplication + # in other vendor sections. + updated_vendor_relationships_map = populate_vendor_relationships_basic_attributes( + vendor_map=updated_vendor_relationships_map, + matching_privacy_declarations=matching_privacy_declarations, + ) + return combine_overlay_sections( purpose_consent_map, # type: ignore[arg-type] purpose_legitimate_interests_map, # type: ignore[arg-type] diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 3040ff7329e..547b7a0f707 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -477,6 +477,11 @@ def system_create_request_body(self) -> SystemSchema: dpo="John Doe, CIPT", joint_controller_info="Jane Doe", data_security_practices="We encrypt all your data in transit and at rest", + cookie_max_age_seconds="31536000", + uses_cookies=True, + cookie_refresh=True, + uses_non_cookie_access=True, + legitimate_interest_disclosure_url="http://www.example.com/legitimate_interest_disclosure", privacy_declarations=[ models.PrivacyDeclaration( name="declaration-name", @@ -654,6 +659,14 @@ async def test_system_create( system.data_security_practices == "We encrypt all your data in transit and at rest" ) + assert system.cookie_max_age_seconds == 31536000 + assert system.uses_cookies is True + assert system.cookie_refresh is True + assert system.uses_non_cookie_access is True + assert ( + system.legitimate_interest_disclosure_url + == "http://www.example.com/legitimate_interest_disclosure" + ) assert system.data_stewards == [] assert [cookie.name for cookie in systems[0].cookies] == ["essential_cookie"] assert [ @@ -679,6 +692,7 @@ async def test_system_create( assert privacy_decl.data_shared_with_third_parties is True assert privacy_decl.third_parties == "Third Party Marketing Dept." assert privacy_decl.shared_categories == ["user"] + assert privacy_decl.flexible_legal_basis_for_processing is None async def test_system_create_minimal_request_body( self, generate_auth_header, db, test_config, system_create_request_body diff --git a/tests/ops/util/test_tcf_experience_contents.py b/tests/ops/util/test_tcf_experience_contents.py index 8c5b4a664db..c092ccff6ea 100644 --- a/tests/ops/util/test_tcf_experience_contents.py +++ b/tests/ops/util/test_tcf_experience_contents.py @@ -146,7 +146,7 @@ def test_feature_that_is_not_in_the_gvl(self, db, system): v_r_len=0, s_c_len=1, s_li_len=0, - s_r_len=0, + s_r_len=1, ) def test_system_has_feature_on_different_declaration_than_relevant_use( @@ -191,6 +191,47 @@ def test_system_has_feature_on_different_declaration_than_relevant_use( s_r_len=0, ) + def test_system_has_declaration_no_features_special_features_special_purposes( + self, tcf_system, db + ): + """Assert that a VendorRelationship record is created even if no features, special features or special purposes are present. + VendorRelationship is still used to store basic Vendor attributes. + """ + decl = tcf_system.privacy_declarations[0] + decl.features = [] + decl.save(db) + + decl_2 = tcf_system.privacy_declarations[1] + decl_2.delete(db) + + tcf_contents = get_tcf_contents(db) + + assert_length_of_tcf_sections( + tcf_contents, + p_c_len=1, + p_li_len=0, + f_len=0, + sp_len=0, + sf_len=0, + v_c_len=1, + v_li_len=0, + v_r_len=1, + s_c_len=0, + s_li_len=0, + s_r_len=0, + ) + + vendor_relationship = tcf_contents.tcf_vendor_relationships[0] + assert vendor_relationship.features == [] + assert vendor_relationship.special_purposes == [] + assert vendor_relationship.special_features == [] + assert vendor_relationship.id == "sendgrid" + assert vendor_relationship.cookie_max_age_seconds is None + assert vendor_relationship.uses_cookies is False + assert vendor_relationship.uses_non_cookie_access is False + assert vendor_relationship.cookie_refresh is False + assert vendor_relationship.legitimate_interest_disclosure_url is None + @pytest.mark.usefixtures("tcf_system") def test_system_exists_with_tcf_purpose_and_vendor(self, db): """System has vendor id so we return preferences against a "vendor" instead of the system""" @@ -224,6 +265,16 @@ def test_system_exists_with_tcf_purpose_and_vendor(self, db): tcf_contents.tcf_vendor_consents[0].description == "My TCF System Description" ) + + # assert some additional TCF attributes are NOT set on the consents object - only on VendorRelationships + assert not hasattr( + tcf_contents.tcf_vendor_consents[0], "cookie_max_age_seconds" + ) + assert not hasattr(tcf_contents.tcf_vendor_consents[0], "uses_cookies") + assert not hasattr( + tcf_contents.tcf_vendor_consents[0], "legitimate_interest_disclosure_url" + ) + assert len(tcf_contents.tcf_vendor_consents[0].purpose_consents) == 1 assert tcf_contents.tcf_vendor_consents[0].purpose_consents[0].id == 8 @@ -233,6 +284,94 @@ def test_system_exists_with_tcf_purpose_and_vendor(self, db): tcf_contents.tcf_vendor_relationships[0].description == "My TCF System Description" ) + # assert some additional TCF attributes are set to their defaults here - this is where they belong! + assert tcf_contents.tcf_vendor_relationships[0].cookie_max_age_seconds is None + assert tcf_contents.tcf_vendor_relationships[0].uses_cookies is False + assert tcf_contents.tcf_vendor_relationships[0].uses_non_cookie_access is False + assert tcf_contents.tcf_vendor_relationships[0].cookie_refresh is False + assert ( + tcf_contents.tcf_vendor_relationships[0].legitimate_interest_disclosure_url + is None + ) + assert len(tcf_contents.tcf_vendor_relationships[0].special_purposes) == 1 + assert tcf_contents.tcf_vendor_relationships[0].special_purposes[0].id == 1 + + def test_system_exists_with_tcf_purpose_and_vendor_including_tcf_fields_set( + self, db, tcf_system + ): + """System has vendor id so we return preferences against a "vendor" instead of the system + + Vendor has some TCF-specific attributes set, ensure they are being surfaced properly in the overlay. + """ + tcf_system.cookie_max_age_seconds = 31536000 + tcf_system.uses_cookies = True + tcf_system.cookie_refresh = True + tcf_system.uses_non_cookie_access = True + tcf_system.legitimate_interest_disclosure_url = "http://test.com/disclosure_url" + tcf_system.save(db) + + tcf_contents = get_tcf_contents(db) + assert_length_of_tcf_sections( + tcf_contents, + p_c_len=1, + p_li_len=0, + f_len=0, + sp_len=1, + sf_len=0, + v_c_len=1, + v_li_len=0, + v_r_len=1, + s_c_len=0, + s_li_len=0, + s_r_len=0, + ) + + assert tcf_contents.tcf_purpose_consents[0].id == 8 + assert tcf_contents.tcf_purpose_consents[0].data_uses == [ + "analytics.reporting.content_performance" + ] + assert tcf_contents.tcf_purpose_consents[0].vendors == [ + EmbeddedVendor(id="sendgrid", name="TCF System Test") + ] + + assert tcf_contents.tcf_vendor_consents[0].id == "sendgrid" + assert tcf_contents.tcf_vendor_consents[0].name == "TCF System Test" + assert ( + tcf_contents.tcf_vendor_consents[0].description + == "My TCF System Description" + ) + + # assert some additional TCF attributes are NOT set on the consents object - only on VendorRelationships + assert not hasattr( + tcf_contents.tcf_vendor_consents[0], "cookie_max_age_seconds" + ) + assert not hasattr(tcf_contents.tcf_vendor_consents[0], "uses_cookies") + assert not hasattr( + tcf_contents.tcf_vendor_consents[0], "legitimate_interest_disclosure_url" + ) + + assert len(tcf_contents.tcf_vendor_consents[0].purpose_consents) == 1 + assert tcf_contents.tcf_vendor_consents[0].purpose_consents[0].id == 8 + + assert tcf_contents.tcf_vendor_relationships[0].id == "sendgrid" + assert tcf_contents.tcf_vendor_relationships[0].name == "TCF System Test" + assert ( + tcf_contents.tcf_vendor_relationships[0].description + == "My TCF System Description" + ) + + # assert some additional TCF attributes are being populated properly based on the System record + assert ( + tcf_contents.tcf_vendor_relationships[0].cookie_max_age_seconds == 31536000 + ) + assert tcf_contents.tcf_vendor_relationships[0].uses_cookies is True + assert tcf_contents.tcf_vendor_relationships[0].uses_non_cookie_access is True + assert tcf_contents.tcf_vendor_relationships[0].cookie_refresh is True + assert ( + tcf_contents.tcf_vendor_relationships[0].legitimate_interest_disclosure_url + == "http://test.com/disclosure_url" + ) + assert len(tcf_contents.tcf_vendor_relationships[0].special_purposes) == 1 assert tcf_contents.tcf_vendor_relationships[0].special_purposes[0].id == 1 @@ -382,7 +521,7 @@ def test_system_matches_subset_of_purpose_data_uses(self, db, tcf_system): sf_len=0, v_c_len=1, v_li_len=0, - v_r_len=0, + v_r_len=1, s_c_len=0, s_li_len=0, s_r_len=0, @@ -559,7 +698,7 @@ def test_system_with_multiple_privacy_declarations(self, db, system): v_r_len=0, s_c_len=1, s_li_len=1, - s_r_len=0, + s_r_len=1, ) first_purpose = tcf_contents.tcf_purpose_consents[0] @@ -596,7 +735,7 @@ def test_duplicate_data_uses_on_system(self, tcf_system, db): sf_len=0, v_c_len=1, v_li_len=1, - v_r_len=0, + v_r_len=1, s_c_len=0, s_li_len=0, s_r_len=0, @@ -661,7 +800,7 @@ def test_add_different_data_uses_that_correspond_to_same_purpose( sf_len=0, v_c_len=1, v_li_len=1, - v_r_len=0, + v_r_len=1, s_c_len=0, s_li_len=0, s_r_len=0, @@ -761,7 +900,7 @@ def test_add_same_data_use_to_different_systems( v_r_len=0, s_c_len=1, s_li_len=2, - s_r_len=0, + s_r_len=2, ) assert len(tcf_contents.tcf_purpose_consents[0].vendors) == 0 assert tcf_contents.tcf_purpose_consents[0].id == 4