Skip to content

Commit

Permalink
First pass at phase 1 TCF 2.0 support (#1228)
Browse files Browse the repository at this point in the history 
* First pass at phase 1 TCF 2.0 support

* minor fixes

* Update go-gdpr library and fix stuff

* Fixes for PR comments
  • Loading branch information
@hhhjort
hhhjort authored Mar 25, 2020
1 parent c7ead07 commit 145c525
Show file tree
Hide file tree
Showing 7 changed files with 122 additions and 59 deletions.
14 changes: 11 additions & 3 deletions gdpr/gdpr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"net/http"

"github.com/prebid/go-gdpr/vendorlist"
"github.com/prebid/prebid-server/config"
"github.com/prebid/prebid-server/openrtb_ext"
)
Expand All @@ -25,6 +26,11 @@ type Permissions interface {
PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, error)
}

const (
tCF1 uint8 = 1
tCF2 uint8 = 2
)

// NewPermissions gets an instance of the Permissions for use elsewhere in the project.
func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_ext.BidderName]uint16, client *http.Client) Permissions {
// If the host doesn't buy into the IAB GDPR consent framework, then save some cycles and let all syncs happen.
Expand All @@ -33,9 +39,11 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
}

return &permissionsImpl{
cfg: cfg,
vendorIDs: vendorIDs,
fetchVendorList: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker),
cfg: cfg,
vendorIDs: vendorIDs,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF1),
tCF2: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF2)},
}
}

Expand Down
28 changes: 20 additions & 8 deletions gdpr/impl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package gdpr
import (
"context"

"github.com/prebid/go-gdpr/consentconstants"
"github.com/prebid/go-gdpr/api"
tcf1constants "github.com/prebid/go-gdpr/consentconstants"
consentconstants "github.com/prebid/go-gdpr/consentconstants/tcf2"
"github.com/prebid/go-gdpr/vendorconsent"
"github.com/prebid/go-gdpr/vendorlist"
"github.com/prebid/prebid-server/config"
Expand All @@ -18,7 +20,7 @@ import (
type permissionsImpl struct {
cfg config.GDPR
vendorIDs map[openrtb_ext.BidderName]uint16
fetchVendorList func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
fetchVendorList map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
}

func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
Expand Down Expand Up @@ -71,10 +73,10 @@ func (p *permissionsImpl) allowSync(ctx context.Context, vendorID uint16, consen
return false, nil
}

// InfoStorageAccess is the same across TCF 1 and TCF 2
if vendor.Purpose(consentconstants.InfoStorageAccess) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && parsedConsent.VendorConsent(vendorID) {
return true, nil
}

return false, nil
}

Expand All @@ -93,14 +95,20 @@ func (p *permissionsImpl) allowPI(ctx context.Context, vendorID uint16, consent
return false, nil
}

if (vendor.Purpose(consentconstants.InfoStorageAccess) || vendor.LegitimateInterest(consentconstants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && (vendor.Purpose(consentconstants.AdSelectionDeliveryReporting) || vendor.LegitimateInterest(consentconstants.AdSelectionDeliveryReporting)) && parsedConsent.PurposeAllowed(consentconstants.AdSelectionDeliveryReporting) && parsedConsent.VendorConsent(vendorID) {
return true, nil
if parsedConsent.Version() == 2 {
// Need to add the location special purpose once the library supports it.
if (vendor.Purpose(consentconstants.InfoStorageAccess) || vendor.LegitimateInterest(consentconstants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && (vendor.Purpose(consentconstants.PersonalizationProfile) || vendor.LegitimateInterest(consentconstants.PersonalizationProfile)) && parsedConsent.PurposeAllowed(consentconstants.PersonalizationProfile) && parsedConsent.VendorConsent(vendorID) {
return true, nil
}
} else {
if (vendor.Purpose(tcf1constants.InfoStorageAccess) || vendor.LegitimateInterest(tcf1constants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(tcf1constants.InfoStorageAccess) && (vendor.Purpose(tcf1constants.AdSelectionDeliveryReporting) || vendor.LegitimateInterest(tcf1constants.AdSelectionDeliveryReporting)) && parsedConsent.PurposeAllowed(tcf1constants.AdSelectionDeliveryReporting) && parsedConsent.VendorConsent(vendorID) {
return true, nil
}
}

return false, nil
}

func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, consent string) (parsedConsent vendorconsent.VendorConsents, vendor vendorlist.Vendor, err error) {
func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, consent string) (parsedConsent api.VendorConsents, vendor api.Vendor, err error) {
parsedConsent, err = vendorconsent.ParseString(consent)
if err != nil {
err = &ErrorMalformedConsent{
Expand All @@ -110,7 +118,11 @@ func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, cons
return
}

vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
version := parsedConsent.Version()
if version < 1 || version > 2 {
return
}
vendorList, err := p.fetchVendorList[version](ctx, parsedConsent.VendorListVersion())
if err != nil {
return
}
Expand Down
63 changes: 46 additions & 17 deletions gdpr/impl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ func TestNoConsentButAllowByDefault(t *testing.T) {
HostVendorID: 3,
UsersyncIfAmbiguous: true,
},
vendorIDs: nil,
fetchVendorList: failedListFetcher,
vendorIDs: nil,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: failedListFetcher,
tCF2: failedListFetcher,
},
}
allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
assertBoolsEqual(t, true, allowSync)
Expand All @@ -35,8 +38,11 @@ func TestNoConsentAndRejectByDefault(t *testing.T) {
HostVendorID: 3,
UsersyncIfAmbiguous: false,
},
vendorIDs: nil,
fetchVendorList: failedListFetcher,
vendorIDs: nil,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: failedListFetcher,
tCF2: failedListFetcher,
},
}
allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
assertBoolsEqual(t, false, allowSync)
Expand All @@ -63,9 +69,14 @@ func TestAllowedSyncs(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABoAAAAAMw")
Expand Down Expand Up @@ -94,9 +105,14 @@ func TestProhibitedPurposes(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABAAAAAAMw")
Expand Down Expand Up @@ -125,9 +141,14 @@ func TestProhibitedVendors(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

allowSync, err := perms.HostCookiesAllowed(context.Background(), "BOS2bx5OS2bx5ABABBAAABoAAAAAFA")
Expand All @@ -144,7 +165,10 @@ func TestMalformedConsent(t *testing.T) {
cfg: config.GDPR{
HostVendorID: 2,
},
fetchVendorList: listFetcher(nil),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(nil),
tCF2: listFetcher(nil),
},
}

sync, err := perms.HostCookiesAllowed(context.Background(), "BON")
Expand All @@ -169,9 +193,14 @@ func TestAllowPersonalInfo(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

// PI needs both purposes to succeed
Expand Down
46 changes: 29 additions & 17 deletions gdpr/vendorlist-fetching.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,35 +11,37 @@ import (
"time"

"github.com/golang/glog"
"github.com/prebid/go-gdpr/api"
"github.com/prebid/go-gdpr/vendorlist"
"github.com/prebid/go-gdpr/vendorlist2"
"github.com/prebid/prebid-server/config"
"golang.org/x/net/context/ctxhttp"
)

type saveVendors func(uint16, vendorlist.VendorList)
type saveVendors func(uint16, api.VendorList)

// This file provides the vendorlist-fetching function for Prebid Server.
//
// For more info, see https://github.com/prebid/prebid-server/issues/504
//
// Nothing in this file is exported. Public APIs can be found in gdpr.go

func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16) string) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, TCFVer uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
// These save and load functions can be used to store & retrieve lists from our cache.
save, load := newVendorListCache()

withTimeout, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout())
defer cancel()
populateCache(withTimeout, client, urlMaker, save)
populateCache(withTimeout, client, urlMaker, save, TCFVer)

saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout())
saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), TCFVer)

return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
list := load(id)
if list != nil {
return list, nil
}
saveOneSometimes(ctx, client, urlMaker(id), save)
saveOneSometimes(ctx, client, urlMaker(id, TCFVer), save)
list = load(id)
if list != nil {
return list, nil
Expand All @@ -49,17 +51,23 @@ func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http
}

// populateCache saves all the known versions of the vendor list for future use.
func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16) string, saver saveVendors) {
latestVersion := saveOne(ctx, client, urlMaker(0), saver)
func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, TCFVer uint8) {
latestVersion := saveOne(ctx, client, urlMaker(0, TCFVer), saver, TCFVer)

for i := uint16(1); i < latestVersion; i++ {
saveOne(ctx, client, urlMaker(i), saver)
saveOne(ctx, client, urlMaker(i, TCFVer), saver, TCFVer)
}
}

// Make a URL which can be used to fetch a given version of the Global Vendor List. If the version is 0,
// this will fetch the latest version.
func vendorListURLMaker(version uint16) string {
func vendorListURLMaker(version uint16, TCFVer uint8) string {
if TCFVer == 2 {
if version == 0 {
return "https://vendorlist.consensu.org/v2/vendor-list.json"
}
return "https://vendorlist.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(version)) + ".json"
}
if version == 0 {
return "https://vendorlist.consensu.org/vendorlist.json"
}
Expand All @@ -71,7 +79,7 @@ func vendorListURLMaker(version uint16) string {
// The goal here is to update quickly when new versions of the VendorList are released, but not wreck
// server performance if a bad CMP starts sending us malformed consent strings that advertize a version
// that doesn't exist yet.
func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
func newOccasionalSaver(timeout time.Duration, TCFVer uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
lastSaved := &atomic.Value{}
lastSaved.Store(time.Time{})

Expand All @@ -80,13 +88,13 @@ func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client
if now.Sub(lastSaved.Load().(time.Time)).Minutes() > 10 {
withTimeout, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
saveOne(withTimeout, client, url, saver)
saveOne(withTimeout, client, url, saver, TCFVer)
lastSaved.Store(now)
}
}
}

func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors) uint16 {
func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, cTFVer uint8) uint16 {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
glog.Errorf("Failed to build GET %s request. Cookie syncs may be affected: %v", url, err)
Expand All @@ -109,8 +117,12 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
glog.Errorf("GET %s returned %d. Cookie syncs may be affected.", url, resp.StatusCode)
return 0
}

newList, err := vendorlist.ParseEagerly(respBody)
var newList api.VendorList
if cTFVer == 2 {
newList, err = vendorlist2.ParseEagerly(respBody)
} else {
newList, err = vendorlist.ParseEagerly(respBody)
}
if err != nil {
glog.Errorf("GET %s returned malformed JSON. Cookie syncs may be affected. Error was %v. Body was %s", url, err, string(respBody))
return 0
Expand All @@ -120,13 +132,13 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
return newList.Version()
}

func newVendorListCache() (save func(id uint16, list vendorlist.VendorList), load func(id uint16) vendorlist.VendorList) {
func newVendorListCache() (save func(id uint16, list api.VendorList), load func(id uint16) api.VendorList) {
cache := &sync.Map{}

save = func(id uint16, list vendorlist.VendorList) {
save = func(id uint16, list api.VendorList) {
cache.Store(id, list)
}
load = func(id uint16) vendorlist.VendorList {
load = func(id uint16) api.VendorList {
list, ok := cache.Load(id)
if ok {
return list.(vendorlist.VendorList)
Expand Down
Loading

0 comments on commit 145c525

Please sign in to comment.