fides: add System model support for new tcf fields (#4228)

Co-authored-by: Dawn Pattison <[email protected]>

.fides/db_dataset.yml

@@ -1142,6 +1142,21 @@ dataset:
     - name: data_security_practices
       data_categories:
       - system.operations
+    - name: cookie_max_age_seconds
+      data_categories:
+      - system.operations
+    - name: cookie_refresh
+      data_categories:
+      - system.operations
+    - name: uses_cookies
+      data_categories:
+      - system.operations
+    - name: uses_non_cookie_access
+      data_categories:
+      - system.operations
+    - name: legitimate_interest_disclosure_url
+      data_categories:
+      - system.operations
     - name: user_id
       data_categories:
       - user.unique_id
@@ -1553,6 +1568,9 @@ dataset:
     - name: legal_basis_for_processing
       data_categories:
       - system.operations
+    - name: flexible_legal_basis_for_processing
+      data_categories:
+      - system.operations
     - name: impact_assessment_location
       data_categories:
       - system.operations

CHANGELOG.md

@@ -21,6 +21,7 @@ The types of changes are:
 - Added an option to link to vendor tab from an experience config description [#4191](https://github.com/ethyca/fides/pull/4191)
 - Added two toggles for vendors in the TCF overlay, one for Consent, and one for Legitimate Interest [#4189](https://github.com/ethyca/fides/pull/4189)
 - Added two toggles for purposes in the TCF overlay, one for Consent, and one for Legitimate Interest [#4234](https://github.com/ethyca/fides/pull/4234)
+- Added support for new TCF-related fields on `System` and `PrivacyDeclaration` models [#4228](https://github.com/ethyca/fides/pull/4228)
 
 
 ### Changed

src/fides/api/alembic/migrations/versions/81886da90395_add_legal_basis_dimension.py

@@ -4,7 +4,7 @@
 need to instead save preferences against a purpose/vendor AND a legal basis.
 
 Revision ID: 81886da90395
-Revises: 4cb3b5af4160
+Revises: 9b98aba5bba8
 Create Date: 2023-09-30 17:39:46.251444
 
 """

src/fides/api/alembic/migrations/versions/c5a218831820_additional_tcf_fields_system.py

@@ -0,0 +1,89 @@
+"""additional tcf fields system
+
+Revision ID: c5a218831820
+Revises: 81226042d7d4
+Create Date: 2023-10-05 15:40:09.294013
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "c5a218831820"
+down_revision = "81226042d7d4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "ctl_systems",
+        sa.Column("cookie_max_age_seconds", sa.Integer(), nullable=True),
+    )
+    op.add_column(
+        "ctl_systems",
+        sa.Column(
+            "uses_cookies",
+            sa.Boolean(),
+            nullable=False,
+            server_default="f",
+        ),
+    )
+    op.add_column(
+        "ctl_systems",
+        sa.Column(
+            "cookie_refresh",
+            sa.Boolean(),
+            nullable=False,
+            server_default="f",
+        ),
+    )
+    op.add_column(
+        "ctl_systems",
+        sa.Column(
+            "uses_non_cookie_access",
+            sa.Boolean(),
+            nullable=False,
+            server_default="f",
+        ),
+    )
+    op.add_column(
+        "ctl_systems",
+        sa.Column("legitimate_interest_disclosure_url", sa.String(), nullable=True),
+    )
+
+    op.add_column(
+        "privacydeclaration",
+        sa.Column(
+            "flexible_legal_basis_for_processing",
+            sa.Boolean(),
+            nullable=True,
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column(
+        "ctl_systems",
+        "cookie_max_age_seconds",
+    )
+    op.drop_column(
+        "ctl_systems",
+        "uses_cookies",
+    )
+    op.drop_column(
+        "ctl_systems",
+        "cookie_refresh",
+    )
+    op.drop_column(
+        "ctl_systems",
+        "uses_non_cookie_access",
+    )
+    op.drop_column(
+        "ctl_systems",
+        "legitimate_interest_disclosure_url",
+    )
+    op.drop_column(
+        "privacydeclaration",
+        "flexible_legal_basis_for_processing",
+    )

src/fides/api/models/sql_models.py

@@ -397,6 +397,15 @@ class System(Base, FidesBase):
     dpo = Column(String)
     joint_controller_info = Column(String)
     data_security_practices = Column(String)
+    cookie_max_age_seconds = Column(Integer)
+    uses_cookies = Column(BOOLEAN(), default=False, server_default="f", nullable=False)
+    cookie_refresh = Column(
+        BOOLEAN(), default=False, server_default="f", nullable=False
+    )
+    uses_non_cookie_access = Column(
+        BOOLEAN(), default=False, server_default="f", nullable=False
+    )
+    legitimate_interest_disclosure_url = Column(String)
 
     privacy_declarations = relationship(
         "PrivacyDeclaration",
@@ -465,6 +474,7 @@ class PrivacyDeclaration(Base):
 
     features = Column(ARRAY(String), server_default="{}", nullable=False)
     legal_basis_for_processing = Column(String)
+    flexible_legal_basis_for_processing = Column(BOOLEAN())
     impact_assessment_location = Column(String)
     retention_period = Column(String)
     processes_special_category_data = Column(

src/fides/api/schemas/tcf.py

@@ -7,7 +7,7 @@
     MAPPED_SPECIAL_PURPOSES,
 )
 from fideslang.gvl.models import Feature, MappedPurpose
-from pydantic import Field, root_validator, validator
+from pydantic import AnyUrl, Field, root_validator, validator
 
 from fides.api.models.privacy_notice import UserConsentPreference
 from fides.api.schemas.base_class import FidesSchema
@@ -110,6 +110,11 @@ class TCFVendorRelationships(CommonVendorFields):
     special_purposes: List[EmbeddedLineItem] = []
     features: List[EmbeddedLineItem] = []
     special_features: List[EmbeddedLineItem] = []
+    cookie_max_age_seconds: Optional[int]
+    uses_cookies: Optional[bool]
+    cookie_refresh: Optional[bool]
+    uses_non_cookie_access: Optional[bool]
+    legitimate_interest_disclosure_url: Optional[AnyUrl]
 
 
 class TCFFeatureRecord(NonVendorSection, Feature):

src/fides/api/util/tcf/tcf_experience_contents.py

@@ -184,6 +184,13 @@ def get_matching_privacy_declarations(db: Session) -> Query:
             System.fides_key.label("system_fides_key"),
             System.name.label("system_name"),
             System.description.label("system_description"),
+            System.cookie_max_age_seconds.label("system_cookie_max_age_seconds"),
+            System.uses_cookies.label("system_uses_cookies"),
+            System.cookie_refresh.label("system_cookie_refresh"),
+            System.uses_non_cookie_access.label("system_uses_non_cookie_access"),
+            System.legitimate_interest_disclosure_url.label(
+                "system_legitimate_interest_disclosure_url"
+            ),
             System.vendor_id,
             PrivacyDeclaration.data_use,
             PrivacyDeclaration.legal_basis_for_processing,
@@ -418,6 +425,50 @@ def build_purpose_or_feature_section_and_update_vendor_map(
     return non_vendor_record_map, vendor_map
 
 
+def populate_vendor_relationships_basic_attributes(
+    vendor_map: Dict[str, TCFVendorRelationships],
+    matching_privacy_declarations: Query,
+) -> Dict[str, TCFVendorRelationships]:
+    """Populates TCFVendorRelationships records for all vendors that we're displaying in the overlay.
+    Ensures that these TCFVendorRelationships records have the "basic" TCF attributes populated.
+    """
+    for privacy_declaration_row in matching_privacy_declarations:
+        vendor_id, system_identifier = get_system_identifiers(privacy_declaration_row)
+
+        # Get the existing TCFVendorRelationships record or create a new one.
+        # Add to the vendor map if it wasn't added in a previous section.
+        vendor_relationship_record = vendor_map.get(system_identifier)
+        if not vendor_relationship_record:
+            vendor_relationship_record = TCFVendorRelationships(
+                id=system_identifier,  # Identify system by vendor id if it exists, otherwise use system id.
+                name=privacy_declaration_row.system_name,
+                description=privacy_declaration_row.system_description,
+                has_vendor_id=bool(
+                    vendor_id  # This will let us separate data between systems and vendors later
+                ),
+            )
+            vendor_map[system_identifier] = vendor_relationship_record
+
+        # Now add basic attributes to the VendorRelationships record
+        vendor_relationship_record.cookie_max_age_seconds = (
+            privacy_declaration_row.system_cookie_max_age_seconds
+        )
+        vendor_relationship_record.uses_cookies = (
+            privacy_declaration_row.system_uses_cookies
+        )
+        vendor_relationship_record.cookie_refresh = (
+            privacy_declaration_row.system_cookie_refresh
+        )
+        vendor_relationship_record.uses_non_cookie_access = (
+            privacy_declaration_row.system_uses_non_cookie_access
+        )
+        vendor_relationship_record.legitimate_interest_disclosure_url = (
+            privacy_declaration_row.system_legitimate_interest_disclosure_url
+        )
+
+    return vendor_map
+
+
 def get_tcf_contents(
     db: Session,
 ) -> TCFExperienceContents:
@@ -511,6 +562,15 @@ def get_tcf_contents(
         matching_privacy_declaration_query=matching_privacy_declarations,
     )
 
+    # Finally, add missing TCFVendorRelationships records for vendors that weren't already added
+    # via the special_features, features, and special_purposes section.  Every vendor in the overlay
+    # should show up in this section. Add the basic attributes to the vendor here to avoid duplication
+    # in other vendor sections.
+    updated_vendor_relationships_map = populate_vendor_relationships_basic_attributes(
+        vendor_map=updated_vendor_relationships_map,
+        matching_privacy_declarations=matching_privacy_declarations,
+    )
+
     return combine_overlay_sections(
         purpose_consent_map,  # type: ignore[arg-type]
         purpose_legitimate_interests_map,  # type: ignore[arg-type]

tests/ctl/core/test_api.py

@@ -477,6 +477,11 @@ def system_create_request_body(self) -> SystemSchema:
             dpo="John Doe, CIPT",
             joint_controller_info="Jane Doe",
             data_security_practices="We encrypt all your data in transit and at rest",
+            cookie_max_age_seconds="31536000",
+            uses_cookies=True,
+            cookie_refresh=True,
+            uses_non_cookie_access=True,
+            legitimate_interest_disclosure_url="http://www.example.com/legitimate_interest_disclosure",
             privacy_declarations=[
                 models.PrivacyDeclaration(
                     name="declaration-name",
@@ -654,6 +659,14 @@ async def test_system_create(
             system.data_security_practices
             == "We encrypt all your data in transit and at rest"
         )
+        assert system.cookie_max_age_seconds == 31536000
+        assert system.uses_cookies is True
+        assert system.cookie_refresh is True
+        assert system.uses_non_cookie_access is True
+        assert (
+            system.legitimate_interest_disclosure_url
+            == "http://www.example.com/legitimate_interest_disclosure"
+        )
         assert system.data_stewards == []
         assert [cookie.name for cookie in systems[0].cookies] == ["essential_cookie"]
         assert [
@@ -679,6 +692,7 @@ async def test_system_create(
         assert privacy_decl.data_shared_with_third_parties is True
         assert privacy_decl.third_parties == "Third Party Marketing Dept."
         assert privacy_decl.shared_categories == ["user"]
+        assert privacy_decl.flexible_legal_basis_for_processing is None
 
     async def test_system_create_minimal_request_body(
         self, generate_auth_header, db, test_config, system_create_request_body