From 1c8e9dd8c5bb06705b64270c225b7e778284e6cc Mon Sep 17 00:00:00 2001 From: Scott Kay Date: Tue, 8 Sep 2020 17:14:52 -0400 Subject: [PATCH] Refactor TCF 1/2 Vendor List Fetcher Tests (#1441) --- gdpr/gdpr.go | 9 +- gdpr/impl_test.go | 183 +++--- gdpr/vendorlist-fetching.go | 118 ++-- gdpr/vendorlist-fetching_test.go | 954 ++++++++++++++++++++++--------- 4 files changed, 851 insertions(+), 413 deletions(-) diff --git a/gdpr/gdpr.go b/gdpr/gdpr.go index 04db8cb92ed..6d447beb438 100644 --- a/gdpr/gdpr.go +++ b/gdpr/gdpr.go @@ -29,9 +29,10 @@ type Permissions interface { AMPException() bool } +// Versions of the GDPR TCF technical specification. const ( - tCF1 uint8 = 1 - tCF2 uint8 = 2 + tcf1SpecVersion uint8 = 1 + tcf2SpecVersion uint8 = 2 ) // NewPermissions gets an instance of the Permissions for use elsewhere in the project. @@ -45,8 +46,8 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_ cfg: cfg, vendorIDs: vendorIDs, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF1), - tCF2: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF2)}, + tcf1SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tcf1SpecVersion), + tcf2SpecVersion: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tcf2SpecVersion)}, } } diff --git a/gdpr/impl_test.go b/gdpr/impl_test.go index 053e87536ab..d5114454f06 100644 --- a/gdpr/impl_test.go +++ b/gdpr/impl_test.go @@ -23,8 +23,8 @@ func TestNoConsentButAllowByDefault(t *testing.T) { }, vendorIDs: nil, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: failedListFetcher, - tCF2: failedListFetcher, + tcf1SpecVersion: failedListFetcher, + tcf2SpecVersion: failedListFetcher, }, } allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "") @@ -43,8 +43,8 @@ func TestNoConsentAndRejectByDefault(t *testing.T) { }, vendorIDs: nil, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: failedListFetcher, - tCF2: failedListFetcher, + tcf1SpecVersion: failedListFetcher, + tcf2SpecVersion: failedListFetcher, }, } allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "") @@ -56,12 +56,11 @@ func TestNoConsentAndRejectByDefault(t *testing.T) { } func TestAllowedSyncs(t *testing.T) { - vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{ - 2: { - purposes: []int{1}, - }, - 3: { - purposes: []int{1}, + vendorListData := tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 1, + Vendors: []tcf1Vendor{ + {ID: 2, Purposes: []int{1}}, + {ID: 3, Purposes: []int{1}}, }, }) perms := permissionsImpl{ @@ -73,10 +72,10 @@ func TestAllowedSyncs(t *testing.T) { openrtb_ext.BidderPubmatic: 3, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), }, @@ -92,12 +91,11 @@ func TestAllowedSyncs(t *testing.T) { } func TestProhibitedPurposes(t *testing.T) { - vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{ - 2: { - purposes: []int{1}, // cookie reads/writes - }, - 3: { - purposes: []int{3}, // ad personalization + vendorListData := tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 1, + Vendors: []tcf1Vendor{ + {ID: 2, Purposes: []int{1}}, // cookie reads/writes + {ID: 3, Purposes: []int{3}}, // ad personalization }, }) perms := permissionsImpl{ @@ -109,10 +107,10 @@ func TestProhibitedPurposes(t *testing.T) { openrtb_ext.BidderPubmatic: 3, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), }, @@ -128,12 +126,11 @@ func TestProhibitedPurposes(t *testing.T) { } func TestProhibitedVendors(t *testing.T) { - vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{ - 2: { - purposes: []int{1}, // cookie reads/writes - }, - 3: { - purposes: []int{3}, // ad personalization + vendorListData := tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 1, + Vendors: []tcf1Vendor{ + {ID: 2, Purposes: []int{1}}, // cookie reads/writes + {ID: 3, Purposes: []int{3}}, // ad personalization }, }) perms := permissionsImpl{ @@ -145,10 +142,10 @@ func TestProhibitedVendors(t *testing.T) { openrtb_ext.BidderPubmatic: 3, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), }, @@ -169,8 +166,8 @@ func TestMalformedConsent(t *testing.T) { HostVendorID: 2, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: listFetcher(nil), - tCF2: listFetcher(nil), + tcf1SpecVersion: listFetcher(nil), + tcf2SpecVersion: listFetcher(nil), }, } @@ -180,12 +177,11 @@ func TestMalformedConsent(t *testing.T) { } func TestAllowPersonalInfo(t *testing.T) { - vendorListData := mockVendorListData(t, 1, map[uint16]*purposes{ - 2: { - purposes: []int{1}, // cookie reads/writes - }, - 3: { - purposes: []int{1, 3}, // ad personalization + vendorListData := tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 1, + Vendors: []tcf1Vendor{ + {ID: 2, Purposes: []int{1}}, // cookie reads/writes + {ID: 3, Purposes: []int{1, 3}}, // ad personalization }, }) perms := permissionsImpl{ @@ -197,10 +193,10 @@ func TestAllowPersonalInfo(t *testing.T) { openrtb_ext.BidderPubmatic: 3, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 1: parseVendorListData(t, vendorListData), }), }, @@ -222,24 +218,39 @@ func TestAllowPersonalInfo(t *testing.T) { assertBoolsEqual(t, true, allowPI) } -var tcf2BasicPurposes = map[uint16]*purposes{ - 2: {purposes: []int{1}}, //cookie reads/writes - 6: {purposes: []int{1, 2, 4}}, // ad personalization - 8: {purposes: []int{1, 7}}, - 10: {purposes: []int{2, 4, 7}}, - 32: {purposes: []int{1, 2, 4, 7}}, -} -var tcf2LegitInterests = map[uint16]*purposes{ - 6: {purposes: []int{7}}, - 8: {purposes: []int{2, 4}}, -} -var tcf2SpecialPuproses = map[uint16]*purposes{ - 6: {purposes: []int{1}}, - 10: {purposes: []int{1}}, -} -var tcf2FlexPurposes = map[uint16]*purposes{ - 6: {purposes: []int{1, 2, 4, 7}}, +func buildTCF2VendorList34() tcf2VendorList { + return tcf2VendorList{ + VendorListVersion: 2, + Vendors: map[string]*tcf2Vendor{ + "2": { + ID: 2, + Purposes: []int{1}, + }, + "6": { + ID: 6, + Purposes: []int{1, 2, 4}, + LegIntPurposes: []int{7}, + SpecialPurposes: []int{1}, + FlexiblePurposes: []int{1, 2, 4, 7}, + }, + "8": { + ID: 8, + Purposes: []int{1, 7}, + LegIntPurposes: []int{2, 4}, + }, + "10": { + ID: 10, + Purposes: []int{2, 4, 7}, + SpecialPurposes: []int{1}, + }, + "32": { + ID: 32, + Purposes: []int{1, 2, 4, 7}, + }, + }, + } } + var tcf2Config = config.GDPR{ HostVendorID: 2, TCF2: config.TCF2{ @@ -261,7 +272,7 @@ type tcf2TestDef struct { } func TestAllowPersonalInfoTCF2(t *testing.T) { - vendorListData := mockVendorListDataTCF2(t, 2, tcf2BasicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -270,8 +281,8 @@ func TestAllowPersonalInfoTCF2(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, @@ -316,7 +327,7 @@ func TestAllowPersonalInfoTCF2(t *testing.T) { } func TestAllowPersonalInfoWhitelistTCF2(t *testing.T) { - vendorListData := mockVendorListDataTCF2(t, 2, tcf2BasicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -325,8 +336,8 @@ func TestAllowPersonalInfoWhitelistTCF2(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, @@ -338,11 +349,10 @@ func TestAllowPersonalInfoWhitelistTCF2(t *testing.T) { assert.EqualValuesf(t, true, allowPI, "AllowPI failure") assert.EqualValuesf(t, true, allowGeo, "AllowGeo failure") assert.EqualValuesf(t, true, allowID, "AllowID failure") - } func TestAllowPersonalInfoTCF2PubRestrict(t *testing.T) { - vendorListData := mockVendorListDataTCF2(t, 2, tcf2BasicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -351,8 +361,8 @@ func TestAllowPersonalInfoTCF2PubRestrict(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 15: parseVendorListDataV2(t, vendorListData), }), }, @@ -397,7 +407,7 @@ func TestAllowPersonalInfoTCF2PubRestrict(t *testing.T) { } func TestAllowPersonalInfoTCF2PurposeOneTrue(t *testing.T) { - vendorListData := mockVendorListDataTCF2(t, 2, tcf2BasicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -406,8 +416,8 @@ func TestAllowPersonalInfoTCF2PurposeOneTrue(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, @@ -453,7 +463,7 @@ func TestAllowPersonalInfoTCF2PurposeOneTrue(t *testing.T) { } func TestAllowPersonalInfoTCF2PurposeOneFalse(t *testing.T) { - vendorListData := mockVendorListDataTCF2(t, 2, tcf2BasicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -462,8 +472,8 @@ func TestAllowPersonalInfoTCF2PurposeOneFalse(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, @@ -510,7 +520,7 @@ func TestAllowPersonalInfoTCF2PurposeOneFalse(t *testing.T) { } func TestAllowSyncTCF2(t *testing.T) { - vendorListData := mockVendorListDataTCF2(t, 2, tcf2BasicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -519,8 +529,8 @@ func TestAllowSyncTCF2(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, @@ -537,9 +547,9 @@ func TestAllowSyncTCF2(t *testing.T) { } func TestProhibitedPurposeSyncTCF2(t *testing.T) { - basicPurposes := tcf2BasicPurposes - basicPurposes[8] = &purposes{purposes: []int{7}} - vendorListData := mockVendorListDataTCF2(t, 2, basicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + tcf2VendorList34 := buildTCF2VendorList34() + tcf2VendorList34.Vendors["8"].Purposes = []int{7} + vendorListData := tcf2MarshalVendorList(tcf2VendorList34) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -548,15 +558,15 @@ func TestProhibitedPurposeSyncTCF2(t *testing.T) { openrtb_ext.BidderRubicon: 8, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, } perms.cfg.HostVendorID = 8 - // COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA : TCF2 with full consensts to purposes and vendors 2, 6, 8 + // COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA : TCF2 with full consents to purposes for vendors 2, 6, 8 allowSync, err := perms.HostCookiesAllowed(context.Background(), "COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA") assert.NoErrorf(t, err, "Error processing HostCookiesAllowed") assert.EqualValuesf(t, false, allowSync, "HostCookiesAllowed failure") @@ -567,9 +577,7 @@ func TestProhibitedPurposeSyncTCF2(t *testing.T) { } func TestProhibitedVendorSyncTCF2(t *testing.T) { - basicPurposes := tcf2BasicPurposes - basicPurposes[10] = &purposes{purposes: []int{1}} - vendorListData := mockVendorListDataTCF2(t, 2, basicPurposes, tcf2LegitInterests, tcf2FlexPurposes, tcf2SpecialPuproses) + vendorListData := tcf2MarshalVendorList(buildTCF2VendorList34()) perms := permissionsImpl{ cfg: tcf2Config, vendorIDs: map[openrtb_ext.BidderName]uint16{ @@ -579,20 +587,21 @@ func TestProhibitedVendorSyncTCF2(t *testing.T) { openrtb_ext.BidderOpenx: 10, }, fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){ - tCF1: nil, - tCF2: listFetcher(map[uint16]vendorlist.VendorList{ + tcf1SpecVersion: nil, + tcf2SpecVersion: listFetcher(map[uint16]vendorlist.VendorList{ 34: parseVendorListDataV2(t, vendorListData), }), }, } perms.cfg.HostVendorID = 10 - // COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA : TCF2 with full consensts to purposes and vendors 2, 4, 6 + // COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA : TCF2 with full consents to purposes for vendors 2, 6, 8 allowSync, err := perms.HostCookiesAllowed(context.Background(), "COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA") assert.NoErrorf(t, err, "Error processing HostCookiesAllowed") assert.EqualValuesf(t, false, allowSync, "HostCookiesAllowed failure") - allowSync, err = perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderRubicon, "COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA") + // Permission disallowed due to consent string not including vendor 10. + allowSync, err = perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderOpenx, "COzTVhaOzTVhaGvAAAENAiCIAP_AAH_AAAAAAEEUACCKAAA") assert.NoErrorf(t, err, "Error processing BidderSyncAllowed") assert.EqualValuesf(t, false, allowSync, "BidderSyncAllowed failure") } diff --git a/gdpr/vendorlist-fetching.go b/gdpr/vendorlist-fetching.go index 1442f81c3ba..66a3f4ad2d6 100644 --- a/gdpr/vendorlist-fetching.go +++ b/gdpr/vendorlist-fetching.go @@ -26,67 +26,83 @@ type saveVendors func(uint16, api.VendorList) // // Nothing in this file is exported. Public APIs can be found in gdpr.go -func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, TCFVer uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) { - var fallbackVL api.VendorList = nil - - if TCFVer == tCF1 && len(cfg.TCF1.FallbackGVLPath) > 0 { - fallbackVL = loadFallbackGVL(cfg.TCF1.FallbackGVLPath) - } - - // If we are not going to try fetching the GVL dynamically, we have a simple fetcher - if !cfg.TCF1.FetchGVL && TCFVer == tCF1 && fallbackVL != nil { - return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) { - return fallbackVL, nil +func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, tcfSpecVersion uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) { + var fallback api.VendorList + if tcfSpecVersion == tcf1SpecVersion && len(cfg.TCF1.FallbackGVLPath) > 0 { + fallback = loadFallbackGVL(cfg.TCF1.FallbackGVLPath) + } + + // If we are not going to try fetching the GVL dynamically, we have a simple fetcher. + if !cfg.TCF1.FetchGVL && tcfSpecVersion == tcf1SpecVersion { + if fallback != nil { + return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) { + return fallback, nil + } + } + return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) { + return nil, makeVendorListNotFoundError(vendorListVersion) } } - // These save and load functions can be used to store & retrieve lists from our cache. - save, load := newVendorListCache(fallbackVL) - withTimeout, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout()) - defer cancel() - populateCache(withTimeout, client, urlMaker, save, TCFVer) + cacheSave, cacheLoad := newVendorListCache(fallback) - saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), TCFVer) + preloadContext, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout()) + defer cancel() + preloadCache(preloadContext, client, urlMaker, cacheSave, tcfSpecVersion) - return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) { - list := load(id) - if list != nil { + saveOneRateLimited := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), tcfSpecVersion) + return func(ctx context.Context, vendorListVersion uint16) (vendorlist.VendorList, error) { + // Attempt To Load From Cache + if list := cacheLoad(vendorListVersion); list != nil { return list, nil } - saveOneSometimes(ctx, client, urlMaker(id, TCFVer), save) - list = load(id) - if list != nil { + + // Attempt To Download + // - May not add to cache immediately. + saveOneRateLimited(ctx, client, urlMaker(vendorListVersion, tcfSpecVersion), cacheSave) + + // Attempt To Load From Cache Again + // - May have been added by the call to saveOneRateLimited. + if list := cacheLoad(vendorListVersion); list != nil { return list, nil } - if fallbackVL != nil { - return fallbackVL, nil + + // Attempt To Use Hardcoded Fallback + if fallback != nil { + return fallback, nil } - return nil, fmt.Errorf("gdpr vendor list version %d does not exist, or has not been loaded yet. Try again in a few minutes", id) + + // Give Up + return nil, makeVendorListNotFoundError(vendorListVersion) } } -// populateCache saves all the known versions of the vendor list for future use. -func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, TCFVer uint8) { - latestVersion := saveOne(ctx, client, urlMaker(0, TCFVer), saver, TCFVer) +func makeVendorListNotFoundError(vendorListVersion uint16) error { + return fmt.Errorf("gdpr vendor list version %d does not exist, or has not been loaded yet. Try again in a few minutes", vendorListVersion) +} + +// preloadCache saves all the known versions of the vendor list for future use. +func preloadCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, tcfSpecVersion uint8) { + latestVersion := saveOne(ctx, client, urlMaker(0, tcfSpecVersion), saver, tcfSpecVersion) for i := uint16(1); i < latestVersion; i++ { - saveOne(ctx, client, urlMaker(i, TCFVer), saver, TCFVer) + saveOne(ctx, client, urlMaker(i, tcfSpecVersion), saver, tcfSpecVersion) } } // Make a URL which can be used to fetch a given version of the Global Vendor List. If the version is 0, // this will fetch the latest version. -func vendorListURLMaker(version uint16, TCFVer uint8) string { - if TCFVer == 2 { - if version == 0 { +func vendorListURLMaker(vendorListVersion uint16, tcfSpecVersion uint8) string { + if tcfSpecVersion == tcf2SpecVersion { + if vendorListVersion == 0 { return "https://vendorlist.consensu.org/v2/vendor-list.json" } - return "https://vendorlist.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(version)) + ".json" + return "https://vendorlist.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(vendorListVersion)) + ".json" } - if version == 0 { + if vendorListVersion == 0 { return "https://vendorlist.consensu.org/vendorlist.json" } - return "https://vendorlist.consensu.org/v-" + strconv.Itoa(int(version)) + "/vendorlist.json" + return "https://vendorlist.consensu.org/v-" + strconv.Itoa(int(vendorListVersion)) + "/vendorlist.json" } // newOccasionalSaver returns a wrapped version of saveOne() which only activates every few minutes. @@ -94,22 +110,24 @@ func vendorListURLMaker(version uint16, TCFVer uint8) string { // The goal here is to update quickly when new versions of the VendorList are released, but not wreck // server performance if a bad CMP starts sending us malformed consent strings that advertize a version // that doesn't exist yet. -func newOccasionalSaver(timeout time.Duration, TCFVer uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) { +func newOccasionalSaver(timeout time.Duration, tcfSpecVersion uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) { lastSaved := &atomic.Value{} lastSaved.Store(time.Time{}) return func(ctx context.Context, client *http.Client, url string, saver saveVendors) { now := time.Now() - if now.Sub(lastSaved.Load().(time.Time)).Minutes() > 10 { + timeSinceLastSave := now.Sub(lastSaved.Load().(time.Time)) + + if timeSinceLastSave.Minutes() > 10 { withTimeout, cancel := context.WithTimeout(ctx, timeout) defer cancel() - saveOne(withTimeout, client, url, saver, TCFVer) + saveOne(withTimeout, client, url, saver, tcfSpecVersion) lastSaved.Store(now) } } } -func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, cTFVer uint8) uint16 { +func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, tcfSpecVersion uint8) uint16 { req, err := http.NewRequest("GET", url, nil) if err != nil { glog.Errorf("Failed to build GET %s request. Cookie syncs may be affected: %v", url, err) @@ -133,7 +151,7 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen return 0 } var newList api.VendorList - if cTFVer == 2 { + if tcfSpecVersion == tcf2SpecVersion { newList, err = vendorlist2.ParseEagerly(respBody) } else { newList, err = vendorlist.ParseEagerly(respBody) @@ -147,14 +165,15 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen return newList.Version() } -func newVendorListCache(fallbackVL api.VendorList) (save func(id uint16, list api.VendorList), load func(id uint16) api.VendorList) { +func newVendorListCache(fallbackVL api.VendorList) (save func(vendorListVersion uint16, list api.VendorList), load func(vendorListVersion uint16) api.VendorList) { cache := &sync.Map{} - save = func(id uint16, list api.VendorList) { - cache.Store(id, list) + save = func(vendorListVersion uint16, list api.VendorList) { + cache.Store(vendorListVersion, list) } - load = func(id uint16) api.VendorList { - list, ok := cache.Load(id) + + load = func(vendorListVersion uint16) api.VendorList { + list, ok := cache.Load(vendorListVersion) if ok { return list.(vendorlist.VendorList) } @@ -164,13 +183,14 @@ func newVendorListCache(fallbackVL api.VendorList) (save func(id uint16, list ap } func loadFallbackGVL(fallbackGVLPath string) vendorlist.VendorList { - fallbackVLbody, err := ioutil.ReadFile(fallbackGVLPath) + fallbackContents, err := ioutil.ReadFile(fallbackGVLPath) if err != nil { glog.Fatalf("Error reading from file %s: %v", fallbackGVLPath, err) } - fallbackVL, err := vendorlist.ParseEagerly(fallbackVLbody) + + fallback, err := vendorlist.ParseEagerly(fallbackContents) if err != nil { glog.Fatalf("Error processing default GVL from %s: %v", fallbackGVLPath, err) } - return fallbackVL + return fallback } diff --git a/gdpr/vendorlist-fetching_test.go b/gdpr/vendorlist-fetching_test.go index 484a0a54b41..e5ad8793b4f 100644 --- a/gdpr/vendorlist-fetching_test.go +++ b/gdpr/vendorlist-fetching_test.go @@ -7,233 +7,697 @@ import ( "net/http/httptest" "strconv" "testing" - "time" "github.com/stretchr/testify/assert" + "github.com/prebid/go-gdpr/consentconstants" "github.com/prebid/prebid-server/config" ) -func TestVendorFetch(t *testing.T) { - vendorListOne := mockVendorListData(t, 1, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, - }, - }) - vendorListTwo := mockVendorListData(t, 2, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2, 3}, +func TestTCF1FetcherInitialLoad(t *testing.T) { + // Loads two vendor lists during initialization by setting the latest vendor list version to 2. + + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 2, + vendorLists: map[int]string{ + 1: tcf1VendorList1, + 2: tcf1VendorList2, }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(2, map[int]string{ - 1: vendorListOne, - 2: vendorListTwo, }))) defer server.Close() - fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), 1) - list, err := fetcher(context.Background(), 1) - assertNilErr(t, err) - vendor := list.Vendor(32) - assertBoolsEqual(t, true, vendor.Purpose(1)) - assertBoolsEqual(t, false, vendor.Purpose(3)) - assertBoolsEqual(t, false, vendor.Purpose(4)) - - list, err = fetcher(context.Background(), 2) - assertNilErr(t, err) - vendor = list.Vendor(32) - assertBoolsEqual(t, true, vendor.Purpose(1)) - assertBoolsEqual(t, true, vendor.Purpose(3)) -} - -func TestLazyFetch(t *testing.T) { - firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, - }, - }) - secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{ - 3: { - purposes: []int{1}, - }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{ - 1: firstVendorList, - 2: secondVendorList, + testCases := []test{ + { + description: "Fetch - No Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 1, + }, + expected: vendorList1Expected, + }, + { + description: "Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "Fetch - Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 1, + }, + expected: vendorList1Expected, + }, + { + description: "Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "No Fetch - Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 1, + }, + expected: vendorListFallbackExpected, + }, + { + description: "No Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorListFallbackExpected, + }, + { + description: "No Fetch - No Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 1, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 1 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + { + description: "No Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + } + + for _, test := range testCases { + runTest(t, test, tcf1SpecVersion, server) + } +} + +func TestTCF2FetcherInitialLoad(t *testing.T) { + // Loads two vendor lists during initialization by setting the latest vendor list version to 2. + // Ensures TCF1 fetch settings have no effect on TCF2. + + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 2, + vendorLists: map[int]string{ + 1: tcf2VendorList1, + 2: tcf2VendorList2, + }, }))) defer server.Close() - fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), 1) - list, err := fetcher(context.Background(), 2) - assertNilErr(t, err) + testCases := []test{ + { + description: "Fetch - No Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 1, + }, + expected: vendorList1Expected, + }, + { + description: "Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "Fetch - Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 1, + }, + expected: vendorList1Expected, + }, + { + description: "Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "No Fetch - Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 1, + }, + expected: vendorList1Expected, + }, + { + description: "No Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "No Fetch - No Fallback - Vendor List 1", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 1, + }, + expected: vendorList1Expected, + }, + { + description: "No Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + } - vendor := list.Vendor(3) - assertBoolsEqual(t, true, vendor.Purpose(1)) - assertBoolsEqual(t, false, vendor.Purpose(2)) + for _, test := range testCases { + runTest(t, test, tcf2SpecVersion, server) + } } -func TestInitialTimeout(t *testing.T) { - list := mockVendorListData(t, 1, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, +func TestTCF1FetcherDynamicLoadListExists(t *testing.T) { + // Loads the first vendor list during initialization by setting the latest vendor list version to 1. + // All other vendor lists will be dynamically loaded. + + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: tcf1VendorList1, + 2: tcf1VendorList2, }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{ - 1: list, }))) defer server.Close() - ctx, cancel := context.WithDeadline(context.Background(), time.Time{}) - defer cancel() - fetcher := newVendorListFetcher(ctx, testConfig(), server.Client(), testURLMaker(server), 1) - _, err := fetcher(context.Background(), 1) // This should do a lazy fetch, even though the initial call failed - assertNilErr(t, err) + testCases := []test{ + { + description: "Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "No Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorListFallbackExpected, + }, + { + description: "No Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + } + + for _, test := range testCases { + runTest(t, test, tcf1SpecVersion, server) + } } -func TestFetchThrottling(t *testing.T) { - vendorListTwo := mockVendorListData(t, 2, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, - }, - }) - vendorListThree := mockVendorListData(t, 3, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, +func TestTCF2FetcherDynamicLoadListExists(t *testing.T) { + // Loads the first vendor list during initialization by setting the latest vendor list version to 1. + // All other vendor lists will be dynamically loaded. + // Ensures TCF1 fetch settings have no effect on TCF2. + + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: tcf2VendorList1, + 2: tcf2VendorList2, }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{ - 1: "{}", - 2: vendorListTwo, - 3: vendorListThree, }))) defer server.Close() - fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), 1) - _, err := fetcher(context.Background(), 2) - assertNilErr(t, err) - _, err = fetcher(context.Background(), 3) - assertErr(t, err, false) + testCases := []test{ + { + description: "Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "No Fetch - Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + { + description: "No Fetch - No Fallback - Vendor List 2", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: vendorList2Expected, + }, + } + + for _, test := range testCases { + runTest(t, test, tcf2SpecVersion, server) + } } -func TestMalformedVendorlistFetch(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{1: "{}"}))) +func TestTCF1FetcherDynamicLoadListDoesntExist(t *testing.T) { + // Loads the first vendor list during initialization by setting the latest vendor list version to 1. + // All other vendor list load attempts will be done dynamically. + + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: tcf1VendorList1, + }, + }))) defer server.Close() - fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), 1) - _, err := fetcher(context.Background(), 1) - assertErr(t, err, false) + testCases := []test{ + { + description: "Fetch - No Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + { + description: "Fetch - Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorListFallbackExpected, + }, + { + description: "No Fetch - Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: vendorListFallbackExpected, + }, + { + description: "No Fetch - No Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + } + + for _, test := range testCases { + runTest(t, test, 1, server) + } } -func TestMissingVendorlistFetch(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{1: "{}"}))) +func TestTCF2FetcherDynamicLoadListDoesntExist(t *testing.T) { + // Loads the first vendor list during initialization by setting the latest vendor list version to 1. + // All other vendor list load attempts will be done dynamically. + // Ensures TCF1 fetch settings have no effect on TCF2. + + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: tcf2VendorList1, + }, + }))) defer server.Close() - fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), 1) - _, err := fetcher(context.Background(), 2) - assertErr(t, err, false) -} + testCases := []test{ + { + description: "Fetch - No Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + { + description: "Fetch - Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: true, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + { + description: "No Fetch - Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: true, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + { + description: "No Fetch - No Fallback - Vendor Doesn't Exist", + setup: testSetup{ + enableTCF1Fetch: false, + enableTCF1Fallback: false, + vendorListVersion: 2, + }, + expected: testExpected{ + errorMessage: "gdpr vendor list version 2 does not exist, or has not been loaded yet. Try again in a few minutes", + }, + }, + } -func TestVendorListMaker(t *testing.T) { - assertStringsEqual(t, "https://vendorlist.consensu.org/vendorlist.json", vendorListURLMaker(0, 1)) - assertStringsEqual(t, "https://vendorlist.consensu.org/v-2/vendorlist.json", vendorListURLMaker(2, 1)) - assertStringsEqual(t, "https://vendorlist.consensu.org/v-12/vendorlist.json", vendorListURLMaker(12, 1)) - assertStringsEqual(t, "https://vendorlist.consensu.org/v2/vendor-list.json", vendorListURLMaker(0, 2)) - assertStringsEqual(t, "https://vendorlist.consensu.org/v2/archives/vendor-list-v7.json", vendorListURLMaker(7, 2)) + for _, test := range testCases { + runTest(t, test, tcf2SpecVersion, server) + } } -func TestDefaultVendorList(t *testing.T) { - firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, - }, - }) - secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{ - 12: { - purposes: []int{2}, +func TestTCF1FetcherThrottling(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 1, + Vendors: []tcf1Vendor{{ID: 12, Purposes: []int{1}}}, + }), + 2: tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 2, + Vendors: []tcf1Vendor{{ID: 12, Purposes: []int{1, 2}}}, + }), + 3: tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 3, + Vendors: []tcf1Vendor{{ID: 12, Purposes: []int{1, 2, 3}}}, + }), }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(2, map[int]string{ - 1: firstVendorList, - 2: secondVendorList, }))) defer server.Close() - testcfg := testConfig() - testcfg.TCF1.FetchGVL = true - testcfg.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json" - fetcher := newVendorListFetcher(context.Background(), testcfg, server.Client(), testURLMaker(server), 1) + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), tcf1SpecVersion) - list, err := fetcher(context.Background(), 12) - assert.NoError(t, err, "Error with fetching default vendorlist: %v", err) - assert.Equal(t, uint16(215), list.Version(), "Expected to fetch default version 215, got %d", list.Version()) + // Dynamically Load List 2 Successfully + _, errList1 := fetcher(context.Background(), 2) + assert.NoError(t, errList1) - // Testing that we got the default vendorlist data, and not the version off the server. - vendor := list.Vendor(12) - assert.Equal(t, true, vendor.Purpose(1)) - assert.Equal(t, false, vendor.Purpose(2)) + // Fail To Load List 3 Due To Rate Limiting + // - The request is rate limited after dynamically list 2. + _, errList2 := fetcher(context.Background(), 3) + assert.EqualError(t, errList2, "gdpr vendor list version 3 does not exist, or has not been loaded yet. Try again in a few minutes") } -func TestFallbackVendorListPassthrough(t *testing.T) { - firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, +func TestTCF2FetcherThrottling(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: tcf2MarshalVendorList(tcf2VendorList{ + VendorListVersion: 1, + Vendors: map[string]*tcf2Vendor{"12": {ID: 12, Purposes: []int{1}}}, + }), + 2: tcf2MarshalVendorList(tcf2VendorList{ + VendorListVersion: 2, + Vendors: map[string]*tcf2Vendor{"12": {ID: 12, Purposes: []int{1, 2}}}, + }), + 3: tcf2MarshalVendorList(tcf2VendorList{ + VendorListVersion: 3, + Vendors: map[string]*tcf2Vendor{"12": {ID: 12, Purposes: []int{1, 2, 3}}}, + }), }, - }) - secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{ - 12: { - purposes: []int{2}, + }))) + defer server.Close() + + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), tcf2SpecVersion) + + // Dynamically Load List 2 Successfully + _, errList1 := fetcher(context.Background(), 2) + assert.NoError(t, errList1) + + // Fail To Load List 3 Due To Rate Limiting + // - The request is rate limited after dynamically list 2. + _, errList2 := fetcher(context.Background(), 3) + assert.EqualError(t, errList2, "gdpr vendor list version 3 does not exist, or has not been loaded yet. Try again in a few minutes") +} + +func TestTCF1MalformedVendorlist(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: "malformed", }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{ - 1: firstVendorList, - 2: secondVendorList, }))) defer server.Close() - testcfg := testConfig() - testcfg.TCF1.FetchGVL = true - testcfg.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json" - fetcher := newVendorListFetcher(context.Background(), testcfg, server.Client(), testURLMaker(server), 1) - list, err := fetcher(context.Background(), 2) - assert.NoError(t, err, "Error with fetching existing vendorlist: %v", err) - assert.Equal(t, uint16(2), list.Version(), "Expected to fetch mock list version 2, got version %d", list.Version()) - - // Testing that we got the testing vendorlist data, and not the default. - vendor := list.Vendor(12) - assert.Equal(t, false, vendor.Purpose(1)) - assert.Equal(t, true, vendor.Purpose(2)) -} - -func TestFallbackVendorListNoFetch(t *testing.T) { - firstVendorList := mockVendorListData(t, 1, map[uint16]*purposes{ - 32: { - purposes: []int{1, 2}, - }, - }) - secondVendorList := mockVendorListData(t, 2, map[uint16]*purposes{ - 12: { - purposes: []int{2}, - }, - }) - server := httptest.NewServer(http.HandlerFunc(mockServer(1, map[int]string{ - 1: firstVendorList, - 2: secondVendorList, + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), tcf1SpecVersion) + _, err := fetcher(context.Background(), 1) + + // Fetching should fail since vendor list could not be unmarshalled. + assert.Error(t, err) +} + +func TestTCF2MalformedVendorlist(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(mockServer(serverSettings{ + vendorListLatestVersion: 1, + vendorLists: map[int]string{ + 1: "malformed", + }, }))) defer server.Close() - testcfg := testConfig() - testcfg.TCF1.FetchGVL = false - testcfg.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json" - fetcher := newVendorListFetcher(context.Background(), testcfg, server.Client(), testURLMaker(server), 1) - list, err := fetcher(context.Background(), 2) - assert.NoError(t, err, "Error with fetching default vendorlist: %v", err) - assert.Equal(t, uint16(215), list.Version(), "Expected to fetch default version 215, got %d", list.Version()) + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), tcf2SpecVersion) + _, err := fetcher(context.Background(), 1) + + // Fetching should fail since vendor list could not be unmarshalled. + assert.Error(t, err) +} + +func TestTCF1ServerUrlInvalid(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + server.Close() + + invalidURLGenerator := func(uint16, uint8) string { return " http://invalid-url-has-leading-whitespace" } - // Testing that we got the default vendorlist data, and not the version off the server. - vendor := list.Vendor(12) - assert.Equal(t, true, vendor.Purpose(1)) - assert.Equal(t, false, vendor.Purpose(2)) + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), invalidURLGenerator, tcf1SpecVersion) + _, err := fetcher(context.Background(), 1) + assert.EqualError(t, err, "gdpr vendor list version 1 does not exist, or has not been loaded yet. Try again in a few minutes") +} + +func TestTCF2ServerUrlInvalid(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + server.Close() + + invalidURLGenerator := func(uint16, uint8) string { return " http://invalid-url-has-leading-whitespace" } + + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), invalidURLGenerator, tcf2SpecVersion) + _, err := fetcher(context.Background(), 1) + + assert.EqualError(t, err, "gdpr vendor list version 1 does not exist, or has not been loaded yet. Try again in a few minutes") +} + +func TestTCF1ServerUnavailable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + server.Close() + + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), tcf1SpecVersion) + _, err := fetcher(context.Background(), 1) + + assert.EqualError(t, err, "gdpr vendor list version 1 does not exist, or has not been loaded yet. Try again in a few minutes") +} + +func TestTCF2ServerUnavailable(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + server.Close() + + fetcher := newVendorListFetcher(context.Background(), testConfig(), server.Client(), testURLMaker(server), tcf2SpecVersion) + _, err := fetcher(context.Background(), 1) + + assert.EqualError(t, err, "gdpr vendor list version 1 does not exist, or has not been loaded yet. Try again in a few minutes") +} + +func TestVendorListURLMaker(t *testing.T) { + testCases := []struct { + description string + tcfSpecVersion uint8 + vendorListVersion uint16 + expectedURL string + }{ + { + description: "TCF1 - Latest", + tcfSpecVersion: 1, + vendorListVersion: 0, // Forces latest version. + expectedURL: "https://vendorlist.consensu.org/vendorlist.json", + }, + { + description: "TCF1 - Specific", + tcfSpecVersion: 1, + vendorListVersion: 42, + expectedURL: "https://vendorlist.consensu.org/v-42/vendorlist.json", + }, + { + description: "TCF2 - Latest", + tcfSpecVersion: 2, + vendorListVersion: 0, // Forces latest version. + expectedURL: "https://vendorlist.consensu.org/v2/vendor-list.json", + }, + { + description: "TCF2 - Specific", + tcfSpecVersion: 2, + vendorListVersion: 42, + expectedURL: "https://vendorlist.consensu.org/v2/archives/vendor-list-v42.json", + }, + } + + for _, test := range testCases { + result := vendorListURLMaker(test.vendorListVersion, test.tcfSpecVersion) + assert.Equal(t, test.expectedURL, result) + } +} + +var tcf1VendorList1 = tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 1, + Vendors: []tcf1Vendor{{ID: 12, Purposes: []int{2}}}, +}) + +var tcf2VendorList1 = tcf2MarshalVendorList(tcf2VendorList{ + VendorListVersion: 1, + Vendors: map[string]*tcf2Vendor{"12": {ID: 12, Purposes: []int{2}}}, +}) + +var vendorList1Expected = testExpected{ + vendorListVersion: 1, + vendorID: 12, + vendorPurposes: map[int]bool{1: false, 2: true, 3: false}, +} + +var tcf1VendorList2 = tcf1MarshalVendorList(tcf1VendorList{ + VendorListVersion: 2, + Vendors: []tcf1Vendor{{ID: 12, Purposes: []int{2, 3}}}, +}) + +var tcf2VendorList2 = tcf2MarshalVendorList(tcf2VendorList{ + VendorListVersion: 2, + Vendors: map[string]*tcf2Vendor{"12": {ID: 12, Purposes: []int{2, 3}}}, +}) + +var vendorList2Expected = testExpected{ + vendorListVersion: 2, + vendorID: 12, + vendorPurposes: map[int]bool{1: false, 2: true, 3: true}, +} + +var vendorListFallbackExpected = testExpected{ + vendorListVersion: 215, // Values from hardcoded fallback file. + vendorID: 12, + vendorPurposes: map[int]bool{1: true, 2: false, 3: true}, +} + +type tcf1VendorList struct { + VendorListVersion uint16 `json:"vendorListVersion"` + Vendors []tcf1Vendor `json:"vendors"` +} + +type tcf1Vendor struct { + ID uint16 `json:"id"` + Purposes []int `json:"purposeIds"` +} + +func tcf1MarshalVendorList(vendorList tcf1VendorList) string { + json, _ := json.Marshal(vendorList) + return string(json) +} + +type tcf2VendorList struct { + VendorListVersion uint16 `json:"vendorListVersion"` + Vendors map[string]*tcf2Vendor `json:"vendors"` +} + +type tcf2Vendor struct { + ID uint16 `json:"id"` + Purposes []int `json:"purposes"` + LegIntPurposes []int `json:"legIntPurposes"` + FlexiblePurposes []int `json:"flexiblePurposes"` + SpecialPurposes []int `json:"specialPurposes"` +} + +func tcf2MarshalVendorList(vendorList tcf2VendorList) string { + json, _ := json.Marshal(vendorList) + return string(json) +} + +type serverSettings struct { + vendorListLatestVersion int + vendorLists map[int]string } // mockServer returns a handler which returns the given response for each global vendor list version. @@ -247,129 +711,74 @@ func TestFallbackVendorListNoFetch(t *testing.T) { // // If the "version" query param points to a version which doesn't exist, it returns a 403. // Don't ask why... that's just what the official page is doing. See https://vendorlist.consensu.org/v-9999/vendorlist.json -func mockServer(latestVersion int, responses map[int]string) func(http.ResponseWriter, *http.Request) { +func mockServer(settings serverSettings) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, req *http.Request) { - version := req.URL.Query().Get("version") - versionInt, err := strconv.Atoi(version) + vendorListVersion := req.URL.Query().Get("version") + vendorListVersionInt, err := strconv.Atoi(vendorListVersion) if err != nil { w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("Request had invalid version: " + version)) + w.Write([]byte("Request had invalid version: " + vendorListVersion)) return } - if versionInt == 0 { - versionInt = latestVersion + if vendorListVersionInt == 0 { + vendorListVersionInt = settings.vendorListLatestVersion } - response, ok := responses[versionInt] + response, ok := settings.vendorLists[vendorListVersionInt] if !ok { w.WriteHeader(http.StatusForbidden) - w.Write([]byte("Version not found: " + version)) + w.Write([]byte("Version not found: " + vendorListVersion)) return } w.Write([]byte(response)) } } -func mockVendorListData(t *testing.T, version uint16, vendors map[uint16]*purposes) string { - type vendorContract struct { - ID uint16 `json:"id"` - Purposes []int `json:"purposeIds"` - } - - type vendorListContract struct { - Version uint16 `json:"vendorListVersion"` - Vendors []vendorContract `json:"vendors"` - } - - buildVendors := func(input map[uint16]*purposes) []vendorContract { - vendors := make([]vendorContract, 0, len(input)) - for id, purpose := range input { - vendors = append(vendors, vendorContract{ - ID: id, - Purposes: purpose.purposes, - }) - } - return vendors - } - - obj := vendorListContract{ - Version: version, - Vendors: buildVendors(vendors), - } - data, err := json.Marshal(obj) - assertNilErr(t, err) - return string(data) +type test struct { + description string + setup testSetup + expected testExpected } -type purposeMap map[uint16]*purposes - -func mockVendorListDataTCF2(t *testing.T, version uint16, basicPurposes purposeMap, legitInterests purposeMap, flexPurposes purposeMap, specialPurposes purposeMap) string { - type vendorContract struct { - ID uint16 `json:"id"` - Purposes []int `json:"purposes"` - LegIntPurposes []int `json:"legIntPurposes"` - FlexiblePurposes []int `json:"flexiblePurposes"` - SpecialPurposes []int `json:"specialPurposes"` - } - - type vendorListContract struct { - Version uint16 `json:"vendorListVersion"` - Vendors map[string]vendorContract `json:"vendors"` - } - - vendors := make(map[string]vendorContract, len(basicPurposes)) - for id, purpose := range basicPurposes { - sid := strconv.Itoa(int(id)) - vendor, ok := vendors[sid] - if !ok { - vendor = vendorContract{ID: id} - } - vendor.Purposes = purpose.purposes - vendors[sid] = vendor - } +type testSetup struct { + enableTCF1Fetch bool + enableTCF1Fallback bool + vendorListVersion uint16 +} - for id, purpose := range legitInterests { - sid := strconv.Itoa(int(id)) - vendor, ok := vendors[sid] - if !ok { - vendor = vendorContract{ID: id} - } - vendor.LegIntPurposes = purpose.purposes - vendors[sid] = vendor - } +type testExpected struct { + errorMessage string + vendorListVersion uint16 + vendorID uint16 + vendorPurposes map[int]bool +} - for id, purpose := range flexPurposes { - sid := strconv.Itoa(int(id)) - vendor, ok := vendors[sid] - if !ok { - vendor = vendorContract{ID: id} - } - vendor.FlexiblePurposes = purpose.purposes - vendors[sid] = vendor +func runTest(t *testing.T, test test, tcfSpecVersion uint8, server *httptest.Server) { + config := testConfig() + config.TCF1.FetchGVL = test.setup.enableTCF1Fetch + if test.setup.enableTCF1Fallback { + config.TCF1.FallbackGVLPath = "../static/tcf1/fallback_gvl.json" } - for id, purpose := range specialPurposes { - sid := strconv.Itoa(int(id)) - vendor, ok := vendors[sid] - if !ok { - vendor = vendorContract{ID: id} + fetcher := newVendorListFetcher(context.Background(), config, server.Client(), testURLMaker(server), tcfSpecVersion) + vendorList, err := fetcher(context.Background(), test.setup.vendorListVersion) + + if test.expected.errorMessage != "" { + assert.EqualError(t, err, test.expected.errorMessage, test.description+":error") + } else { + assert.NoError(t, err, test.description+":vendorlist") + assert.Equal(t, test.expected.vendorListVersion, vendorList.Version(), test.description+":vendorlistid") + vendor := vendorList.Vendor(test.expected.vendorID) + for id, expected := range test.expected.vendorPurposes { + result := vendor.Purpose(consentconstants.Purpose(id)) + assert.Equalf(t, expected, result, "%s:vendor-%d:purpose-%d", test.description, vendorList.Version(), id) } - vendor.SpecialPurposes = purpose.purposes - vendors[sid] = vendor } - - obj := vendorListContract{ - Version: version, - Vendors: vendors, - } - data, err := json.Marshal(obj) - assertNilErr(t, err) - return string(data) } func testURLMaker(server *httptest.Server) func(uint16, uint8) string { url := server.URL - return func(version uint16, TCFVer uint8) string { - return url + "?version=" + strconv.Itoa(int(version)) + return func(vendorListVersion uint16, tcfSpecVersion uint8) string { + return url + "?version=" + strconv.Itoa(int(vendorListVersion)) } } @@ -379,9 +788,8 @@ func testConfig() config.GDPR { InitVendorlistFetch: 60 * 1000, ActiveVendorlistFetch: 1000 * 5, }, + TCF1: config.TCF1{ + FetchGVL: true, + }, } } - -type purposes struct { - purposes []int -}