Add support for Account configurations (PBID-727,prebid#1402)

- Defines the Account object with per-publisher settings - Adds a cfg.DefaultAccount for global settings that are overrideable per publisher - Expands account validation to require a valid account not just any non-empty value - Makes the resolved account object available to auction and analytics - Cache TTL can now be controlled at account level - Uses fetcher framework to retrieve accounts from individual <id>.json files or HTTP API (database to be added later) Example configuration for accounts stored in `stored_requests/data/by_id/accounts/<id>.json` ```yaml accounts: filesystem: enabled: true directorypath: "./stored_requests/data/by_id" ```
openx · Aug 22, 2020 · 2e984f1 · 2e984f1
1 parent 80d557e
commit 2e984f1
Show file tree

Hide file tree

Showing 24 changed files with 314 additions and 51 deletions.
diff --git a/analytics/core.go b/analytics/core.go
@@ -2,6 +2,7 @@ package analytics
 
 import (
 	"github.com/mxmCherry/openrtb"
+	"github.com/prebid/prebid-server/config"
 	"github.com/prebid/prebid-server/openrtb_ext"
 	"github.com/prebid/prebid-server/usersync"
 )
@@ -28,6 +29,7 @@ type AuctionObject struct {
 	Errors   []error
 	Request  *openrtb.BidRequest
 	Response *openrtb.BidResponse
+	Account  *config.Account
 }
 
 //Loggable object of a transaction at /openrtb2/amp endpoint

diff --git a/config/accounts.go b/config/accounts.go
@@ -0,0 +1,14 @@
+package config
+
+// Account represents a publisher account configuration
+type Account struct {
+	ID               string       `mapstructure:"id" json:"id"`
+	Disabled         bool         `mapstructure:"disabled" json:"disabled"`
+	PriceGranularity string       `mapstructure:"price_granularity" json:"price_granularity"`
+	CacheTTL         DefaultTTLs  `mapstructure:"cache_ttl" json:"cache_ttl"`
+	Analytics        PubAnalytics `mapstructure:"analytics" json:"analytics"`
+}
+
+// PubAnalytics contains analytics settings for an account
+type PubAnalytics struct {
+}
diff --git a/config/config.go b/config/config.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/url"
@@ -39,6 +40,8 @@ type Configuration struct {
 	DataCache       DataCache          `mapstructure:"datacache"`
 	StoredRequests  StoredRequests     `mapstructure:"stored_requests"`
 	CategoryMapping StoredRequestsSlim `mapstructure:"category_mapping"`
+	Accounts        StoredRequestsSlim `mapstructure:"accounts"`
+	DefaultAccount  Account            `mapstructure:"default_account"`
 	// Note that StoredVideo refers to stored video requests, and has nothing to do with caching video creatives.
 	StoredVideo StoredRequestsSlim `mapstructure:"stored_video_req"`
 
@@ -578,6 +581,12 @@ func New(v *viper.Viper) (*Configuration, error) {
 		return nil, err
 	}
 
+	// Migrate global settings to default account
+	if c.AccountRequired == false {
+		c.DefaultAccount.Disabled = false
+	}
+	c.DefaultAccount.TTL = c.CacheURL.DefaultTTLs
+
 	// To look for a request's publisher_id in the NonStandardPublishers list in
 	// O(1) time, we fill this hash table located in the NonStandardPublisherMap field of GDPR
 	c.GDPR.NonStandardPublisherMap = make(map[string]int)
@@ -592,6 +601,9 @@ func New(v *viper.Viper) (*Configuration, error) {
 		c.BlacklistedAppMap[c.BlacklistedApps[i]] = true
 	}
 
+	if len(c.BlacklistedAccts) > 0 && c.BlacklistedAccts[0] != "" {
+		glog.Warningf(`Instead of blacklisted_accts, consider using account setting "disabled: true" or global setting "account_required: true".`)
+	}
 	// To look for a request's account id in O(1) time, we fill this hash table located in the
 	// the BlacklistedAccts field of the Configuration struct defined in this file
 	c.BlacklistedAcctMap = make(map[string]bool)

diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go
@@ -44,6 +44,7 @@ func NewAmpEndpoint(
 	ex exchange.Exchange,
 	validator openrtb_ext.BidderParamValidator,
 	requestsById stored_requests.Fetcher,
+	accounts stored_requests.AccountFetcher,
 	categories stored_requests.CategoryFetcher,
 	cfg *config.Configuration,
 	met pbsmetrics.MetricsEngine,
@@ -53,7 +54,7 @@ func NewAmpEndpoint(
 	bidderMap map[string]openrtb_ext.BidderName,
 ) (httprouter.Handle, error) {
 
-	if ex == nil || validator == nil || requestsById == nil || cfg == nil || met == nil {
+	if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil {
 		return nil, errors.New("NewAmpEndpoint requires non-nil arguments.")
 	}
 
@@ -69,6 +70,7 @@ func NewAmpEndpoint(
 		validator,
 		requestsById,
 		empty_fetcher.EmptyFetcher{},
+		accounts,
 		categories,
 		cfg,
 		met,
@@ -155,8 +157,9 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h
 		labels.CookieFlag = pbsmetrics.CookieFlagYes
 	}
 	labels.PubID = effectivePubID(req.Site.Publisher)
-	// Blacklist account now that we have resolved the value
-	if acctIdErr := validateAccount(deps.cfg, labels.PubID); acctIdErr != nil {
+	// Look up account now that we have resolved the pubID value
+	account, acctIdErr := deps.getAccount(ctx, labels.PubID)
+	if acctIdErr != nil {
 		errL = append(errL, acctIdErr)
 		errCode := errortypes.ReadCode(acctIdErr)
 		if errCode == errortypes.BlacklistedAppErrorCode || errCode == errortypes.BlacklistedAcctErrorCode {
@@ -173,7 +176,7 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h
 		return
 	}
 
-	response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, &deps.categories, nil)
+	response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, account, &deps.categories, nil)
 	ao.AuctionResponse = response
 
 	if err != nil {

diff --git a/endpoints/openrtb2/amp_auction_test.go b/endpoints/openrtb2/amp_auction_test.go
@@ -47,6 +47,7 @@ func TestGoodAmpRequests(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{goodRequests},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -100,6 +101,7 @@ func TestAMPPageInfo(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{stored},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -205,6 +207,7 @@ func TestGDPRConsent(t *testing.T) {
 			newParamsValidator(t),
 			&mockAmpStoredReqFetcher{stored},
 			empty_fetcher.EmptyFetcher{},
+			empty_fetcher.EmptyFetcher{},
 			&config.Configuration{MaxRequestSize: maxSize},
 			metrics,
 			analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -358,6 +361,7 @@ func TestCCPAConsent(t *testing.T) {
 			newParamsValidator(t),
 			&mockAmpStoredReqFetcher{stored},
 			empty_fetcher.EmptyFetcher{},
+			empty_fetcher.EmptyFetcher{},
 			&config.Configuration{MaxRequestSize: maxSize},
 			metrics,
 			analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -417,6 +421,7 @@ func TestNoConsent(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{stored},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		metrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -463,6 +468,7 @@ func TestInvalidConsent(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{stored},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		metrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -547,6 +553,7 @@ func TestNewAndLegacyConsentBothProvided(t *testing.T) {
 			newParamsValidator(t),
 			&mockAmpStoredReqFetcher{stored},
 			empty_fetcher.EmptyFetcher{},
+			empty_fetcher.EmptyFetcher{},
 			&config.Configuration{MaxRequestSize: maxSize},
 			metrics,
 			analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -599,6 +606,7 @@ func TestAMPSiteExt(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{stored},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -639,6 +647,7 @@ func TestAmpBadRequests(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{badRequests},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -670,6 +679,7 @@ func TestAmpDebug(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{requests},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -743,6 +753,7 @@ func TestQueryParamOverrides(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{requests},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -895,6 +906,7 @@ func (s formatOverrideSpec) execute(t *testing.T) {
 		newParamsValidator(t),
 		&mockAmpStoredReqFetcher{requests},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),
@@ -952,7 +964,7 @@ var expectedErrorsFromHoldAuction map[openrtb_ext.BidderName][]openrtb_ext.ExtBi
 	},
 }
 
-func (m *mockAmpExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) {
+func (m *mockAmpExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) {
 	m.lastRequest = bidRequest
 
 	response := &openrtb.BidResponse{

diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go
@@ -45,9 +45,9 @@ var (
 	dntEnabled  int8   = 1
 )
 
-func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, disabledBidders map[string]string, defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName) (httprouter.Handle, error) {
+func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, accounts stored_requests.AccountFetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, disabledBidders map[string]string, defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName) (httprouter.Handle, error) {
 
-	if ex == nil || validator == nil || requestsById == nil || cfg == nil || met == nil {
+	if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil {
 		return nil, errors.New("NewEndpoint requires non-nil arguments.")
 	}
 
@@ -63,6 +63,7 @@ func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidato
 		validator,
 		requestsById,
 		empty_fetcher.EmptyFetcher{},
+		accounts,
 		categories,
 		cfg,
 		met,
@@ -81,6 +82,7 @@ type endpointDeps struct {
 	paramsValidator           openrtb_ext.BidderParamValidator
 	storedReqFetcher          stored_requests.Fetcher
 	videoFetcher              stored_requests.Fetcher
+	accounts                  stored_requests.AccountFetcher
 	categories                stored_requests.CategoryFetcher
 	cfg                       *config.Configuration
 	metricsEngine             pbsmetrics.MetricsEngine
@@ -152,15 +154,18 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http
 		labels.PubID = effectivePubID(req.Site.Publisher)
 	}
 
-	if acctIdErr := validateAccount(deps.cfg, labels.PubID); acctIdErr != nil {
+	// Look up account now that we have resolved the pubID value
+	account, acctIdErr := deps.getAccount(ctx, labels.PubID)
+	if acctIdErr != nil {
 		errL = append(errL, acctIdErr)
 		writeError(errL, w, &labels)
 		return
 	}
 
-	response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, &deps.categories, nil)
+	response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, account, &deps.categories, nil)
 	ao.Request = req
 	ao.Response = response
+	ao.Account = account
 	if err != nil {
 		labels.RequestStatus = pbsmetrics.RequestStatusErr
 		w.WriteHeader(http.StatusInternalServerError)
@@ -1282,14 +1287,33 @@ func effectivePubID(pub *openrtb.Publisher) string {
 	return pbsmetrics.PublisherUnknown
 }
 
-func validateAccount(cfg *config.Configuration, pubID string) error {
-	var err error = nil
-	if cfg.AccountRequired && pubID == pbsmetrics.PublisherUnknown {
-		// If specified in the configuration, discard requests that don't come with an account ID.
-		err = error(&errortypes.AcctRequired{Message: fmt.Sprintf("Prebid-server has been configured to discard requests that don't come with an Account ID. Please reach out to the prebid server host.")})
-	} else if _, found := cfg.BlacklistedAcctMap[pubID]; found {
-		// Blacklist account now that we have resolved the value
-		err = error(&errortypes.BlacklistedAcct{Message: fmt.Sprintf("Prebid-server has blacklisted Account ID: %s, please reach out to the prebid server host.", pubID)})
+func (deps *endpointDeps) getAccount(ctx context.Context, pubID string) (*config.Account, error) {
+	// Check BlacklistedAcctMap until we have deprecated it
+	if _, found := deps.cfg.BlacklistedAcctMap[pubID]; found {
+		return nil, error(&errortypes.BlacklistedAcct{
+			Message: fmt.Sprintf("Prebid-server has disabled Account ID: %s, please reach out to the prebid server host.", pubID),
+		})
 	}
-	return err
+	var account *config.Account
+	var err error
+	if account, err = deps.accounts.FetchAccount(ctx, pubID); err != nil || account == nil {
+		if deps.cfg.DefaultAccount.Disabled || (deps.cfg.AccountRequired && pubID == pbsmetrics.PublisherUnknown) {
+			err = error(&errortypes.AcctRequired{
+				Message: fmt.Sprintf("Prebid-server has been configured to discard requests without a valid Account ID. Please reach out to the prebid server host."),
+			})
+		}
+		// We have an unknown account, so use the DefaultAccount
+		// Transitional: make a copy of DefaultAccount instead of taking a reference,
+		// to preserve original pubID in case is needed to check NonStandardPublisherMap
+		pubAccount := deps.cfg.DefaultAccount
+		pubAccount.ID = pubID
+		account = &pubAccount
+	} else {
+		if account.Disabled {
+			err = error(&errortypes.BlacklistedAcct{
+				Message: fmt.Sprintf("Prebid-server has disabled Account ID: %s, please reach out to the prebid server host.", pubID),
+			})
+		}
+	}
+	return account, err
 }
diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go
@@ -83,6 +83,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) {
 		paramValidator,
 		empty_fetcher.EmptyFetcher{},
 		empty_fetcher.EmptyFetcher{},
+		empty_fetcher.EmptyFetcher{},
 		&config.Configuration{MaxRequestSize: maxSize},
 		theMetrics,
 		analyticsConf.NewPBSAnalytics(&config.Analytics{}),