Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

First pass at phase 1 TCF 2.0 support #1228

Merged
merged 4 commits into from
Mar 25, 2020
Merged

First pass at phase 1 TCF 2.0 support #1228

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c4cd2b0
First pass at phase 1 TCF 2.0 support
hhhjort Mar 19, 2020
e91b86e
minor fixes
hhhjort Mar 20, 2020
421203b
Update go-gdpr library and fix stuff
hhhjort Mar 24, 2020
84e1d03
Fixes for PR comments
hhhjort Mar 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions gdpr/gdpr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"net/http"

"github.com/prebid/go-gdpr/vendorlist"
"github.com/prebid/prebid-server/config"
"github.com/prebid/prebid-server/openrtb_ext"
)
Expand All @@ -25,6 +26,11 @@ type Permissions interface {
PersonalInfoAllowed(ctx context.Context, bidder openrtb_ext.BidderName, PublisherID string, consent string) (bool, error)
}

const (
tCF1 uint8 = 1
tCF2 uint8 = 2
)

// NewPermissions gets an instance of the Permissions for use elsewhere in the project.
func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_ext.BidderName]uint16, client *http.Client) Permissions {
// If the host doesn't buy into the IAB GDPR consent framework, then save some cycles and let all syncs happen.
Expand All @@ -33,9 +39,11 @@ func NewPermissions(ctx context.Context, cfg config.GDPR, vendorIDs map[openrtb_
}

return &permissionsImpl{
cfg: cfg,
vendorIDs: vendorIDs,
fetchVendorList: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker),
cfg: cfg,
vendorIDs: vendorIDs,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF1),
tCF2: newVendorListFetcher(ctx, cfg, client, vendorListURLMaker, tCF2)},
}
}

Expand Down
28 changes: 20 additions & 8 deletions gdpr/impl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package gdpr
import (
"context"

"github.com/prebid/go-gdpr/consentconstants"
"github.com/prebid/go-gdpr/api"
tcf1constants "github.com/prebid/go-gdpr/consentconstants"
consentconstants "github.com/prebid/go-gdpr/consentconstants/tcf2"
"github.com/prebid/go-gdpr/vendorconsent"
"github.com/prebid/go-gdpr/vendorlist"
"github.com/prebid/prebid-server/config"
Expand All @@ -18,7 +20,7 @@ import (
type permissionsImpl struct {
cfg config.GDPR
vendorIDs map[openrtb_ext.BidderName]uint16
fetchVendorList func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
fetchVendorList map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error)
}

func (p *permissionsImpl) HostCookiesAllowed(ctx context.Context, consent string) (bool, error) {
Expand Down Expand Up @@ -71,10 +73,10 @@ func (p *permissionsImpl) allowSync(ctx context.Context, vendorID uint16, consen
return false, nil
}

// InfoStorageAccess is the same across TCF 1 and TCF 2
if vendor.Purpose(consentconstants.InfoStorageAccess) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && parsedConsent.VendorConsent(vendorID) {
return true, nil
}

return false, nil
}

Expand All @@ -93,14 +95,20 @@ func (p *permissionsImpl) allowPI(ctx context.Context, vendorID uint16, consent
return false, nil
}

if (vendor.Purpose(consentconstants.InfoStorageAccess) || vendor.LegitimateInterest(consentconstants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && (vendor.Purpose(consentconstants.AdSelectionDeliveryReporting) || vendor.LegitimateInterest(consentconstants.AdSelectionDeliveryReporting)) && parsedConsent.PurposeAllowed(consentconstants.AdSelectionDeliveryReporting) && parsedConsent.VendorConsent(vendorID) {
return true, nil
if parsedConsent.Version() == 2 {
// Need to add the location special purpose once the library supports it.
if (vendor.Purpose(consentconstants.InfoStorageAccess) || vendor.LegitimateInterest(consentconstants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(consentconstants.InfoStorageAccess) && (vendor.Purpose(consentconstants.PersonalizationProfile) || vendor.LegitimateInterest(consentconstants.PersonalizationProfile)) && parsedConsent.PurposeAllowed(consentconstants.PersonalizationProfile) && parsedConsent.VendorConsent(vendorID) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this code have test coverage?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Discussed in person. There are no tests for the TCF 1 version either. This needs to be addressed and can be done in a separate PR. We'll manually run end to end tests to verify functionality for now.

return true, nil
}
} else {
if (vendor.Purpose(tcf1constants.InfoStorageAccess) || vendor.LegitimateInterest(tcf1constants.InfoStorageAccess)) && parsedConsent.PurposeAllowed(tcf1constants.InfoStorageAccess) && (vendor.Purpose(tcf1constants.AdSelectionDeliveryReporting) || vendor.LegitimateInterest(tcf1constants.AdSelectionDeliveryReporting)) && parsedConsent.PurposeAllowed(tcf1constants.AdSelectionDeliveryReporting) && parsedConsent.VendorConsent(vendorID) {
return true, nil
}
}

return false, nil
}

func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, consent string) (parsedConsent vendorconsent.VendorConsents, vendor vendorlist.Vendor, err error) {
func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, consent string) (parsedConsent api.VendorConsents, vendor api.Vendor, err error) {
parsedConsent, err = vendorconsent.ParseString(consent)
if err != nil {
err = &ErrorMalformedConsent{
Expand All @@ -110,7 +118,11 @@ func (p *permissionsImpl) parseVendor(ctx context.Context, vendorID uint16, cons
return
}

vendorList, err := p.fetchVendorList(ctx, parsedConsent.VendorListVersion())
version := parsedConsent.Version()
if version < 1 || version > 2 {
return
}
vendorList, err := p.fetchVendorList[version](ctx, parsedConsent.VendorListVersion())
if err != nil {
return
}
Expand Down
63 changes: 46 additions & 17 deletions gdpr/impl_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ func TestNoConsentButAllowByDefault(t *testing.T) {
HostVendorID: 3,
UsersyncIfAmbiguous: true,
},
vendorIDs: nil,
fetchVendorList: failedListFetcher,
vendorIDs: nil,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: failedListFetcher,
tCF2: failedListFetcher,
},
}
allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
assertBoolsEqual(t, true, allowSync)
Expand All @@ -35,8 +38,11 @@ func TestNoConsentAndRejectByDefault(t *testing.T) {
HostVendorID: 3,
UsersyncIfAmbiguous: false,
},
vendorIDs: nil,
fetchVendorList: failedListFetcher,
vendorIDs: nil,
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: failedListFetcher,
tCF2: failedListFetcher,
},
}
allowSync, err := perms.BidderSyncAllowed(context.Background(), openrtb_ext.BidderAppnexus, "")
assertBoolsEqual(t, false, allowSync)
Expand All @@ -63,9 +69,14 @@ func TestAllowedSyncs(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABoAAAAAMw")
Expand Down Expand Up @@ -94,9 +105,14 @@ func TestProhibitedPurposes(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

allowSync, err := perms.HostCookiesAllowed(context.Background(), "BON3PCUON3PCUABABBAAABAAAAAAMw")
Expand Down Expand Up @@ -125,9 +141,14 @@ func TestProhibitedVendors(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

allowSync, err := perms.HostCookiesAllowed(context.Background(), "BOS2bx5OS2bx5ABABBAAABoAAAAAFA")
Expand All @@ -144,7 +165,10 @@ func TestMalformedConsent(t *testing.T) {
cfg: config.GDPR{
HostVendorID: 2,
},
fetchVendorList: listFetcher(nil),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(nil),
tCF2: listFetcher(nil),
},
}

sync, err := perms.HostCookiesAllowed(context.Background(), "BON")
Expand All @@ -169,9 +193,14 @@ func TestAllowPersonalInfo(t *testing.T) {
openrtb_ext.BidderAppnexus: 2,
openrtb_ext.BidderPubmatic: 3,
},
fetchVendorList: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
fetchVendorList: map[uint8]func(ctx context.Context, id uint16) (vendorlist.VendorList, error){
tCF1: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
tCF2: listFetcher(map[uint16]vendorlist.VendorList{
1: parseVendorListData(t, vendorListData),
}),
},
}

// PI needs both purposes to succeed
Expand Down
46 changes: 29 additions & 17 deletions gdpr/vendorlist-fetching.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,35 +11,37 @@ import (
"time"

"github.com/golang/glog"
"github.com/prebid/go-gdpr/api"
"github.com/prebid/go-gdpr/vendorlist"
"github.com/prebid/go-gdpr/vendorlist2"
"github.com/prebid/prebid-server/config"
"golang.org/x/net/context/ctxhttp"
)

type saveVendors func(uint16, vendorlist.VendorList)
type saveVendors func(uint16, api.VendorList)

// This file provides the vendorlist-fetching function for Prebid Server.
//
// For more info, see https://github.com/prebid/prebid-server/issues/504
//
// Nothing in this file is exported. Public APIs can be found in gdpr.go

func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16) string) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http.Client, urlMaker func(uint16, uint8) string, TCFVer uint8) func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
// These save and load functions can be used to store & retrieve lists from our cache.
save, load := newVendorListCache()

withTimeout, cancel := context.WithTimeout(initCtx, cfg.Timeouts.InitTimeout())
defer cancel()
populateCache(withTimeout, client, urlMaker, save)
populateCache(withTimeout, client, urlMaker, save, TCFVer)

saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout())
saveOneSometimes := newOccasionalSaver(cfg.Timeouts.ActiveTimeout(), TCFVer)

return func(ctx context.Context, id uint16) (vendorlist.VendorList, error) {
list := load(id)
if list != nil {
return list, nil
}
saveOneSometimes(ctx, client, urlMaker(id), save)
saveOneSometimes(ctx, client, urlMaker(id, TCFVer), save)
list = load(id)
if list != nil {
return list, nil
Expand All @@ -49,17 +51,23 @@ func newVendorListFetcher(initCtx context.Context, cfg config.GDPR, client *http
}

// populateCache saves all the known versions of the vendor list for future use.
func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16) string, saver saveVendors) {
latestVersion := saveOne(ctx, client, urlMaker(0), saver)
func populateCache(ctx context.Context, client *http.Client, urlMaker func(uint16, uint8) string, saver saveVendors, TCFVer uint8) {
latestVersion := saveOne(ctx, client, urlMaker(0, TCFVer), saver, TCFVer)

for i := uint16(1); i < latestVersion; i++ {
saveOne(ctx, client, urlMaker(i), saver)
saveOne(ctx, client, urlMaker(i, TCFVer), saver, TCFVer)
}
}

// Make a URL which can be used to fetch a given version of the Global Vendor List. If the version is 0,
// this will fetch the latest version.
func vendorListURLMaker(version uint16) string {
func vendorListURLMaker(version uint16, TCFVer uint8) string {
if TCFVer == 2 {
if version == 0 {
return "https://vendorlist.consensu.org/v2/vendor-list.json"
}
return "https://vendorlist.consensu.org/v2/archives/vendor-list-v" + strconv.Itoa(int(version)) + ".json"
}
if version == 0 {
return "https://vendorlist.consensu.org/vendorlist.json"
}
Expand All @@ -71,7 +79,7 @@ func vendorListURLMaker(version uint16) string {
// The goal here is to update quickly when new versions of the VendorList are released, but not wreck
// server performance if a bad CMP starts sending us malformed consent strings that advertize a version
// that doesn't exist yet.
func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
func newOccasionalSaver(timeout time.Duration, TCFVer uint8) func(ctx context.Context, client *http.Client, url string, saver saveVendors) {
lastSaved := &atomic.Value{}
guscarreon marked this conversation as resolved.
Show resolved Hide resolved
lastSaved.Store(time.Time{})

Expand All @@ -80,13 +88,13 @@ func newOccasionalSaver(timeout time.Duration) func(ctx context.Context, client
if now.Sub(lastSaved.Load().(time.Time)).Minutes() > 10 {
withTimeout, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
saveOne(withTimeout, client, url, saver)
saveOne(withTimeout, client, url, saver, TCFVer)
lastSaved.Store(now)
}
}
}

func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors) uint16 {
func saveOne(ctx context.Context, client *http.Client, url string, saver saveVendors, cTFVer uint8) uint16 {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
glog.Errorf("Failed to build GET %s request. Cookie syncs may be affected: %v", url, err)
Expand All @@ -109,8 +117,12 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
glog.Errorf("GET %s returned %d. Cookie syncs may be affected.", url, resp.StatusCode)
return 0
}

newList, err := vendorlist.ParseEagerly(respBody)
var newList api.VendorList
if cTFVer == 2 {
newList, err = vendorlist2.ParseEagerly(respBody)
} else {
newList, err = vendorlist.ParseEagerly(respBody)
}
if err != nil {
glog.Errorf("GET %s returned malformed JSON. Cookie syncs may be affected. Error was %v. Body was %s", url, err, string(respBody))
return 0
Expand All @@ -120,13 +132,13 @@ func saveOne(ctx context.Context, client *http.Client, url string, saver saveVen
return newList.Version()
}

func newVendorListCache() (save func(id uint16, list vendorlist.VendorList), load func(id uint16) vendorlist.VendorList) {
func newVendorListCache() (save func(id uint16, list api.VendorList), load func(id uint16) api.VendorList) {
cache := &sync.Map{}

save = func(id uint16, list vendorlist.VendorList) {
save = func(id uint16, list api.VendorList) {
cache.Store(id, list)
}
load = func(id uint16) vendorlist.VendorList {
load = func(id uint16) api.VendorList {
list, ok := cache.Load(id)
if ok {
return list.(vendorlist.VendorList)
Expand Down
Loading