diff --git a/analytics/core.go b/analytics/core.go index 6fd5139fd3d..737d133487e 100644 --- a/analytics/core.go +++ b/analytics/core.go @@ -2,6 +2,7 @@ package analytics import ( "github.com/mxmCherry/openrtb" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/openrtb_ext" "github.com/prebid/prebid-server/usersync" ) @@ -28,6 +29,7 @@ type AuctionObject struct { Errors []error Request *openrtb.BidRequest Response *openrtb.BidResponse + Account *config.Account } //Loggable object of a transaction at /openrtb2/amp endpoint diff --git a/config/accounts.go b/config/accounts.go new file mode 100644 index 00000000000..af0eb7b78e6 --- /dev/null +++ b/config/accounts.go @@ -0,0 +1,14 @@ +package config + +// Account represents a publisher account configuration +type Account struct { + ID string `mapstructure:"id" json:"id"` + Disabled bool `mapstructure:"disabled" json:"disabled"` + PriceGranularity string `mapstructure:"price_granularity" json:"price_granularity"` + CacheTTL DefaultTTLs `mapstructure:"cache_ttl" json:"cache_ttl"` + Analytics PubAnalytics `mapstructure:"analytics" json:"analytics"` +} + +// PubAnalytics contains analytics settings for an account +type PubAnalytics struct { +} diff --git a/config/config.go b/config/config.go index e3b7d8ebda0..f58d1504383 100755 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "encoding/json" "errors" "fmt" "net/url" @@ -39,6 +40,8 @@ type Configuration struct { DataCache DataCache `mapstructure:"datacache"` StoredRequests StoredRequests `mapstructure:"stored_requests"` CategoryMapping StoredRequestsSlim `mapstructure:"category_mapping"` + Accounts StoredRequestsSlim `mapstructure:"accounts"` + DefaultAccount Account `mapstructure:"default_account"` // Note that StoredVideo refers to stored video requests, and has nothing to do with caching video creatives. StoredVideo StoredRequestsSlim `mapstructure:"stored_video_req"` @@ -578,6 +581,12 @@ func New(v *viper.Viper) (*Configuration, error) { return nil, err } + // Migrate global settings to default account + if c.AccountRequired == false { + c.DefaultAccount.Disabled = false + } + c.DefaultAccount.TTL = c.CacheURL.DefaultTTLs + // To look for a request's publisher_id in the NonStandardPublishers list in // O(1) time, we fill this hash table located in the NonStandardPublisherMap field of GDPR c.GDPR.NonStandardPublisherMap = make(map[string]int) @@ -592,6 +601,9 @@ func New(v *viper.Viper) (*Configuration, error) { c.BlacklistedAppMap[c.BlacklistedApps[i]] = true } + if len(c.BlacklistedAccts) > 0 && c.BlacklistedAccts[0] != "" { + glog.Warningf(`Instead of blacklisted_accts, consider using account setting "disabled: true" or global setting "account_required: true".`) + } // To look for a request's account id in O(1) time, we fill this hash table located in the // the BlacklistedAccts field of the Configuration struct defined in this file c.BlacklistedAcctMap = make(map[string]bool) diff --git a/endpoints/openrtb2/amp_auction.go b/endpoints/openrtb2/amp_auction.go index 8efba5a926c..d54ffde9d4f 100644 --- a/endpoints/openrtb2/amp_auction.go +++ b/endpoints/openrtb2/amp_auction.go @@ -44,6 +44,7 @@ func NewAmpEndpoint( ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, + accounts stored_requests.AccountFetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, @@ -53,7 +54,7 @@ func NewAmpEndpoint( bidderMap map[string]openrtb_ext.BidderName, ) (httprouter.Handle, error) { - if ex == nil || validator == nil || requestsById == nil || cfg == nil || met == nil { + if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil { return nil, errors.New("NewAmpEndpoint requires non-nil arguments.") } @@ -69,6 +70,7 @@ func NewAmpEndpoint( validator, requestsById, empty_fetcher.EmptyFetcher{}, + accounts, categories, cfg, met, @@ -155,8 +157,9 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h labels.CookieFlag = pbsmetrics.CookieFlagYes } labels.PubID = effectivePubID(req.Site.Publisher) - // Blacklist account now that we have resolved the value - if acctIdErr := validateAccount(deps.cfg, labels.PubID); acctIdErr != nil { + // Look up account now that we have resolved the pubID value + account, acctIdErr := deps.getAccount(ctx, labels.PubID) + if acctIdErr != nil { errL = append(errL, acctIdErr) errCode := errortypes.ReadCode(acctIdErr) if errCode == errortypes.BlacklistedAppErrorCode || errCode == errortypes.BlacklistedAcctErrorCode { @@ -173,7 +176,7 @@ func (deps *endpointDeps) AmpAuction(w http.ResponseWriter, r *http.Request, _ h return } - response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, &deps.categories, nil) + response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, account, &deps.categories, nil) ao.AuctionResponse = response if err != nil { diff --git a/endpoints/openrtb2/amp_auction_test.go b/endpoints/openrtb2/amp_auction_test.go index 692d3fb0c5d..d009be75722 100644 --- a/endpoints/openrtb2/amp_auction_test.go +++ b/endpoints/openrtb2/amp_auction_test.go @@ -47,6 +47,7 @@ func TestGoodAmpRequests(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{goodRequests}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -100,6 +101,7 @@ func TestAMPPageInfo(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -205,6 +207,7 @@ func TestGDPRConsent(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -358,6 +361,7 @@ func TestCCPAConsent(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -417,6 +421,7 @@ func TestNoConsent(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -463,6 +468,7 @@ func TestInvalidConsent(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -547,6 +553,7 @@ func TestNewAndLegacyConsentBothProvided(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -599,6 +606,7 @@ func TestAMPSiteExt(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{stored}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -639,6 +647,7 @@ func TestAmpBadRequests(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{badRequests}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -670,6 +679,7 @@ func TestAmpDebug(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{requests}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -743,6 +753,7 @@ func TestQueryParamOverrides(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{requests}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -895,6 +906,7 @@ func (s formatOverrideSpec) execute(t *testing.T) { newParamsValidator(t), &mockAmpStoredReqFetcher{requests}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -952,7 +964,7 @@ var expectedErrorsFromHoldAuction map[openrtb_ext.BidderName][]openrtb_ext.ExtBi }, } -func (m *mockAmpExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { +func (m *mockAmpExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { m.lastRequest = bidRequest response := &openrtb.BidResponse{ diff --git a/endpoints/openrtb2/auction.go b/endpoints/openrtb2/auction.go index 86186fa8373..924f2c04222 100644 --- a/endpoints/openrtb2/auction.go +++ b/endpoints/openrtb2/auction.go @@ -45,9 +45,9 @@ var ( dntEnabled int8 = 1 ) -func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, disabledBidders map[string]string, defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName) (httprouter.Handle, error) { +func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, accounts stored_requests.AccountFetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, disabledBidders map[string]string, defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName) (httprouter.Handle, error) { - if ex == nil || validator == nil || requestsById == nil || cfg == nil || met == nil { + if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil { return nil, errors.New("NewEndpoint requires non-nil arguments.") } @@ -63,6 +63,7 @@ func NewEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidato validator, requestsById, empty_fetcher.EmptyFetcher{}, + accounts, categories, cfg, met, @@ -81,6 +82,7 @@ type endpointDeps struct { paramsValidator openrtb_ext.BidderParamValidator storedReqFetcher stored_requests.Fetcher videoFetcher stored_requests.Fetcher + accounts stored_requests.AccountFetcher categories stored_requests.CategoryFetcher cfg *config.Configuration metricsEngine pbsmetrics.MetricsEngine @@ -152,15 +154,18 @@ func (deps *endpointDeps) Auction(w http.ResponseWriter, r *http.Request, _ http labels.PubID = effectivePubID(req.Site.Publisher) } - if acctIdErr := validateAccount(deps.cfg, labels.PubID); acctIdErr != nil { + // Look up account now that we have resolved the pubID value + account, acctIdErr := deps.getAccount(ctx, labels.PubID) + if acctIdErr != nil { errL = append(errL, acctIdErr) writeError(errL, w, &labels) return } - response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, &deps.categories, nil) + response, err := deps.ex.HoldAuction(ctx, req, usersyncs, labels, account, &deps.categories, nil) ao.Request = req ao.Response = response + ao.Account = account if err != nil { labels.RequestStatus = pbsmetrics.RequestStatusErr w.WriteHeader(http.StatusInternalServerError) @@ -1282,14 +1287,33 @@ func effectivePubID(pub *openrtb.Publisher) string { return pbsmetrics.PublisherUnknown } -func validateAccount(cfg *config.Configuration, pubID string) error { - var err error = nil - if cfg.AccountRequired && pubID == pbsmetrics.PublisherUnknown { - // If specified in the configuration, discard requests that don't come with an account ID. - err = error(&errortypes.AcctRequired{Message: fmt.Sprintf("Prebid-server has been configured to discard requests that don't come with an Account ID. Please reach out to the prebid server host.")}) - } else if _, found := cfg.BlacklistedAcctMap[pubID]; found { - // Blacklist account now that we have resolved the value - err = error(&errortypes.BlacklistedAcct{Message: fmt.Sprintf("Prebid-server has blacklisted Account ID: %s, please reach out to the prebid server host.", pubID)}) +func (deps *endpointDeps) getAccount(ctx context.Context, pubID string) (*config.Account, error) { + // Check BlacklistedAcctMap until we have deprecated it + if _, found := deps.cfg.BlacklistedAcctMap[pubID]; found { + return nil, error(&errortypes.BlacklistedAcct{ + Message: fmt.Sprintf("Prebid-server has disabled Account ID: %s, please reach out to the prebid server host.", pubID), + }) } - return err + var account *config.Account + var err error + if account, err = deps.accounts.FetchAccount(ctx, pubID); err != nil || account == nil { + if deps.cfg.DefaultAccount.Disabled || (deps.cfg.AccountRequired && pubID == pbsmetrics.PublisherUnknown) { + err = error(&errortypes.AcctRequired{ + Message: fmt.Sprintf("Prebid-server has been configured to discard requests without a valid Account ID. Please reach out to the prebid server host."), + }) + } + // We have an unknown account, so use the DefaultAccount + // Transitional: make a copy of DefaultAccount instead of taking a reference, + // to preserve original pubID in case is needed to check NonStandardPublisherMap + pubAccount := deps.cfg.DefaultAccount + pubAccount.ID = pubID + account = &pubAccount + } else { + if account.Disabled { + err = error(&errortypes.BlacklistedAcct{ + Message: fmt.Sprintf("Prebid-server has disabled Account ID: %s, please reach out to the prebid server host.", pubID), + }) + } + } + return account, err } diff --git a/endpoints/openrtb2/auction_benchmark_test.go b/endpoints/openrtb2/auction_benchmark_test.go index fba0daecea8..09af23af103 100644 --- a/endpoints/openrtb2/auction_benchmark_test.go +++ b/endpoints/openrtb2/auction_benchmark_test.go @@ -83,6 +83,7 @@ func BenchmarkOpenrtbEndpoint(b *testing.B) { paramValidator, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 957760c61c9..f0e4558b4d8 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -103,7 +103,7 @@ func TestExplicitUserId(t *testing.T) { // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) - endpoint, _ := NewEndpoint(ex, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) + endpoint, _ := NewEndpoint(ex, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) endpoint(httptest.NewRecorder(), request, nil) @@ -255,11 +255,21 @@ func TestRejectAccountRequired(t *testing.T) { accountReq: true, }, { - // Account is required, was provided and is not in the blacklisted accounts map + // Account is required, was provided, not blacklisted, but is not valid (not defined by host) dir: "sample-requests/account-required", file: "with-acct.json", payloadGetter: getRequestPayload, messageGetter: nilReturner, + expectedCode: http.StatusBadRequest, + aliased: true, + accountReq: true, + }, + { + // Account is required, was provided, not blacklisted and is a valid account + dir: "sample-requests/account-required", + file: "valid-acct.json", + payloadGetter: getRequestPayload, + messageGetter: nilReturner, expectedCode: http.StatusOK, aliased: true, accountReq: true, @@ -346,12 +356,22 @@ func (gr *getResponseFromDirectory) doRequest(t *testing.T, requestData []byte) // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) + cfg := config.Configuration{ + MaxRequestSize: maxSize, + BlacklistedApps: []string{"spam_app"}, + BlacklistedAppMap: map[string]bool{"spam_app": true}, + BlacklistedAccts: []string{"bad_acct"}, + BlacklistedAcctMap: map[string]bool{"bad_acct": true}, + AccountRequired: gr.accountReq, + DefaultAccount: config.Account{Disabled: gr.accountReq}, + } endpoint, _ := NewEndpoint( &nobidExchange{}, newParamsValidator(t), &mockStoredReqFetcher{}, + &mockAccountFetcher{}, empty_fetcher.EmptyFetcher{}, - &config.Configuration{MaxRequestSize: maxSize, BlacklistedApps: []string{"spam_app"}, BlacklistedAppMap: map[string]bool{"spam_app": true}, BlacklistedAccts: []string{"bad_acct"}, BlacklistedAcctMap: map[string]bool{"bad_acct": true}, AccountRequired: gr.accountReq}, + &cfg, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), disabledBidders, @@ -390,7 +410,7 @@ func doBadAliasRequest(t *testing.T, filename string, expectMsg string) { // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) - endpoint, _ := NewEndpoint(&nobidExchange{}, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), disabledBidders, aliasJSON, bidderMap) + endpoint, _ := NewEndpoint(&nobidExchange{}, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), disabledBidders, aliasJSON, bidderMap) request := httptest.NewRequest("POST", "/openrtb2/auction", bytes.NewReader(requestData)) recorder := httptest.NewRecorder() @@ -454,7 +474,7 @@ func TestNilExchange(t *testing.T) { // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) - _, err := NewEndpoint(nil, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) + _, err := NewEndpoint(nil, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) if err == nil { t.Errorf("NewEndpoint should return an error when given a nil Exchange.") } @@ -465,7 +485,7 @@ func TestNilValidator(t *testing.T) { // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) - _, err := NewEndpoint(&nobidExchange{}, nil, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) + _, err := NewEndpoint(&nobidExchange{}, nil, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) if err == nil { t.Errorf("NewEndpoint should return an error when given a nil BidderParamValidator.") } @@ -476,7 +496,7 @@ func TestExchangeError(t *testing.T) { // NewMetrics() will create a new go_metrics MetricsEngine, bypassing the need for a crafted configuration set to support it. // As a side effect this gives us some coverage of the go_metrics piece of the metrics engine. theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) - endpoint, _ := NewEndpoint(&brokenExchange{}, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) + endpoint, _ := NewEndpoint(&brokenExchange{}, newParamsValidator(t), empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) request := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, "site.json"))) recorder := httptest.NewRecorder() endpoint(recorder, request, nil) @@ -588,7 +608,7 @@ func TestImplicitIPsEndToEnd(t *testing.T) { IPv6PrivateNetworksParsed: test.privateNetworksIPv6, }, } - endpoint, _ := NewEndpoint(exchange, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) + endpoint, _ := NewEndpoint(exchange, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, cfg, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) httpReq := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, test.reqJSONFile))) httpReq.Header.Set("X-Forwarded-For", test.xForwardedForHeader) @@ -773,7 +793,7 @@ func TestImplicitDNTEndToEnd(t *testing.T) { metrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) for _, test := range testCases { exchange := &nobidExchange{} - endpoint, _ := NewEndpoint(exchange, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) + endpoint, _ := NewEndpoint(exchange, newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, metrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), map[string]string{}, []byte{}, openrtb_ext.BidderMap) httpReq := httptest.NewRequest("POST", "/openrtb2/auction", strings.NewReader(validRequest(t, test.reqJSONFile))) httpReq.Header.Set("DNT", test.dntHeader) @@ -846,6 +866,7 @@ func TestStoredRequests(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -885,6 +906,7 @@ func TestOversizedRequest(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(len(reqBody) - 1)}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -920,6 +942,7 @@ func TestRequestSizeEdgeCase(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(len(reqBody))}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -953,6 +976,7 @@ func TestNoEncoding(t *testing.T) { newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1028,6 +1052,7 @@ func TestContentType(t *testing.T) { newParamsValidator(t), &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1058,6 +1083,7 @@ func TestDisabledBidder(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{ MaxRequestSize: int64(len(reqBody)), }, @@ -1096,6 +1122,7 @@ func TestValidateImpExtDisabledBidder(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: int64(8096)}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1137,6 +1164,7 @@ func TestCurrencyTrunc(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1181,6 +1209,7 @@ func TestCCPAInvalid(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1229,6 +1258,7 @@ func TestSChainInvalid(t *testing.T) { &mockStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{}, pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}), analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1349,7 +1379,7 @@ type nobidExchange struct { gotRequest *openrtb.BidRequest } -func (e *nobidExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { +func (e *nobidExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { e.gotRequest = bidRequest return &openrtb.BidResponse{ ID: bidRequest.ID, @@ -1360,7 +1390,7 @@ func (e *nobidExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.Bid type brokenExchange struct{} -func (e *brokenExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { +func (e *brokenExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { return nil, errors.New("Critical, unrecoverable error.") } @@ -1716,11 +1746,24 @@ func (cf mockStoredReqFetcher) FetchRequests(ctx context.Context, requestIDs []s return testStoredRequestData, testStoredImpData, nil } +var mockAccountData = map[string]*config.Account{ + "valid_acct": { + ID: "valid_acct", + }, +} + +type mockAccountFetcher struct { +} + +func (af mockAccountFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + return mockAccountData[accountID], nil +} + type mockExchange struct { lastRequest *openrtb.BidRequest } -func (m *mockExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { +func (m *mockExchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { m.lastRequest = bidRequest return &openrtb.BidResponse{ SeatBid: []openrtb.SeatBid{{ diff --git a/endpoints/openrtb2/sample-requests/account-required/valid-acct.json b/endpoints/openrtb2/sample-requests/account-required/valid-acct.json new file mode 100644 index 00000000000..87d9d1eac37 --- /dev/null +++ b/endpoints/openrtb2/sample-requests/account-required/valid-acct.json @@ -0,0 +1,67 @@ +{ + "description": "This request comes with a valid account id", + "message": "", + + "requestPayload": { + "id": "some-request-id", + "site": { + "publisher": { "id": "valid_acct"}, + "page": "test.somepage.com" + }, + "user": { }, + "imp": [ + { + "id": "my-imp-id", + "banner": { + "format": [ + { + "w": 300, + "h": 600 + } + ] + }, + "pmp": { + "deals": [ + { + "id": "some-deal-id" + } + ] + }, + "ext": { + "appnexus": { + "placementId": 12883451 + } + } + } + ], + "tmax": 500, + "ext": { + "prebid": { + "aliases": { + "districtm": "appnexus" + }, + "bidadjustmentfactors": { + "appnexus": 1.01, + "districtm": 0.98, + "rubicon": 0.99 + }, + "cache": { + "bids": {} + }, + "targeting": { + "includewinners": false, + "pricegranularity": { + "precision": 2, + "ranges": [ + { + "max": 20, + "increment": 0.10 + } + ] + } + } + } + } + } + } + diff --git a/endpoints/openrtb2/video_auction.go b/endpoints/openrtb2/video_auction.go index a6ca527874a..5227c79c879 100644 --- a/endpoints/openrtb2/video_auction.go +++ b/endpoints/openrtb2/video_auction.go @@ -35,9 +35,9 @@ import ( var defaultRequestTimeout int64 = 5000 -func NewVideoEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, videoFetcher stored_requests.Fetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, disabledBidders map[string]string, defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName, cache prebid_cache_client.Client) (httprouter.Handle, error) { +func NewVideoEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamValidator, requestsById stored_requests.Fetcher, videoFetcher stored_requests.Fetcher, accounts stored_requests.AccountFetcher, categories stored_requests.CategoryFetcher, cfg *config.Configuration, met pbsmetrics.MetricsEngine, pbsAnalytics analytics.PBSAnalyticsModule, disabledBidders map[string]string, defReqJSON []byte, bidderMap map[string]openrtb_ext.BidderName, cache prebid_cache_client.Client) (httprouter.Handle, error) { - if ex == nil || validator == nil || requestsById == nil || cfg == nil || met == nil { + if ex == nil || validator == nil || requestsById == nil || accounts == nil || cfg == nil || met == nil { return nil, errors.New("NewVideoEndpoint requires non-nil arguments.") } @@ -55,6 +55,7 @@ func NewVideoEndpoint(ex exchange.Exchange, validator openrtb_ext.BidderParamVal validator, requestsById, videoFetcher, + accounts, categories, cfg, met, @@ -253,13 +254,15 @@ func (deps *endpointDeps) VideoAuctionEndpoint(w http.ResponseWriter, r *http.Re labels.PubID = effectivePubID(bidReq.Site.Publisher) } - if acctIdErr := validateAccount(deps.cfg, labels.PubID); acctIdErr != nil { + // Look up account now that we have resolved the pubID value + account, acctIdErr := deps.getAccount(ctx, labels.PubID) + if acctIdErr != nil { errL := []error{err} handleError(&labels, w, errL, &vo, &debugLog) return } //execute auction logic - response, err := deps.ex.HoldAuction(ctx, bidReq, usersyncs, labels, &deps.categories, &debugLog) + response, err := deps.ex.HoldAuction(ctx, bidReq, usersyncs, labels, account, &deps.categories, &debugLog) vo.Request = bidReq vo.Response = response if err != nil { diff --git a/endpoints/openrtb2/video_auction_test.go b/endpoints/openrtb2/video_auction_test.go index 534db3c79e2..8ad9e46cb15 100644 --- a/endpoints/openrtb2/video_auction_test.go +++ b/endpoints/openrtb2/video_auction_test.go @@ -1168,6 +1168,7 @@ func mockDepsWithMetrics(t *testing.T, ex *mockExchangeVideo) (*endpointDeps, *p &mockVideoStoredReqFetcher{}, &mockVideoStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, mockModule, @@ -1210,6 +1211,7 @@ func mockDeps(t *testing.T, ex *mockExchangeVideo) *endpointDeps { &mockVideoStoredReqFetcher{}, &mockVideoStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1233,6 +1235,7 @@ func mockDepsNoBids(t *testing.T, ex *mockExchangeVideoNoBids) *endpointDeps { &mockVideoStoredReqFetcher{}, &mockVideoStoredReqFetcher{}, empty_fetcher.EmptyFetcher{}, + empty_fetcher.EmptyFetcher{}, &config.Configuration{MaxRequestSize: maxSize}, theMetrics, analyticsConf.NewPBSAnalytics(&config.Analytics{}), @@ -1275,7 +1278,7 @@ type mockExchangeVideo struct { cache *mockCacheClient } -func (m *mockExchangeVideo) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { +func (m *mockExchangeVideo) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { m.lastRequest = bidRequest if debugLog != nil && debugLog.Enabled { m.cache.called = true @@ -1311,7 +1314,7 @@ type mockExchangeVideoNoBids struct { cache *mockCacheClient } -func (m *mockExchangeVideoNoBids) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { +func (m *mockExchangeVideoNoBids) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, ids exchange.IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *exchange.DebugLog) (*openrtb.BidResponse, error) { m.lastRequest = bidRequest return &openrtb.BidResponse{ SeatBid: []openrtb.SeatBid{{}}, diff --git a/exchange/exchange.go b/exchange/exchange.go index e465a78389b..49c032899bf 100644 --- a/exchange/exchange.go +++ b/exchange/exchange.go @@ -36,7 +36,7 @@ const DebugContextKey = ContextKey("debugInfo") // Exchange runs Auctions. Implementations must be threadsafe, and will be shared across many goroutines. type Exchange interface { // HoldAuction executes an OpenRTB v2.5 Auction. - HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, usersyncs IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *DebugLog) (*openrtb.BidResponse, error) + HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, usersyncs IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *DebugLog) (*openrtb.BidResponse, error) } // IdFetcher can find the user's ID for a specific Bidder. @@ -53,7 +53,6 @@ type exchange struct { gDPR gdpr.Permissions currencyConverter *currencies.RateConverter UsersyncIfAmbiguous bool - defaultTTLs config.DefaultTTLs privacyConfig config.Privacy eeaCountries map[string]struct{} } @@ -88,7 +87,6 @@ func NewExchange(client *http.Client, cache prebid_cache_client.Client, cfg *con e.gDPR = gDPR e.currencyConverter = currencyConverter e.UsersyncIfAmbiguous = cfg.GDPR.UsersyncIfAmbiguous - e.defaultTTLs = cfg.CacheURL.DefaultTTLs e.privacyConfig = config.Privacy{ CCPA: cfg.CCPA, GDPR: cfg.GDPR, @@ -97,7 +95,7 @@ func NewExchange(client *http.Client, cache prebid_cache_client.Client, cfg *con return e } -func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, usersyncs IdFetcher, labels pbsmetrics.Labels, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *DebugLog) (*openrtb.BidResponse, error) { +func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidRequest, usersyncs IdFetcher, labels pbsmetrics.Labels, account *config.Account, categoriesFetcher *stored_requests.CategoryFetcher, debugLog *DebugLog) (*openrtb.BidResponse, error) { requestExt, err := extractBidRequestExt(bidRequest) if err != nil { @@ -202,7 +200,7 @@ func (e *exchange) HoldAuction(ctx context.Context, bidRequest *openrtb.BidReque } } - cacheErrs := auc.doCache(ctx, e.cache, targData, bidRequest, 60, &e.defaultTTLs, bidCategory, debugLog) + cacheErrs := auc.doCache(ctx, e.cache, targData, bidRequest, 60, &account.CacheTTL, bidCategory, debugLog) if len(cacheErrs) > 0 { errs = append(errs, cacheErrs...) } diff --git a/exchange/exchange_test.go b/exchange/exchange_test.go index a6f69f70c59..74fb6675490 100644 --- a/exchange/exchange_test.go +++ b/exchange/exchange_test.go @@ -248,7 +248,7 @@ func TestDebugBehaviour(t *testing.T) { } // Run test - outBidResponse, err := e.HoldAuction(context.Background(), bidRequest, &emptyUsersync{}, pbsmetrics.Labels{}, &categoriesFetcher, nil) + outBidResponse, err := e.HoldAuction(context.Background(), bidRequest, &emptyUsersync{}, pbsmetrics.Labels{}, &config.Account{}, &categoriesFetcher, nil) // Assert no HoldAuction error assert.NoErrorf(t, err, "%s. ex.HoldAuction returned an error: %v \n", test.desc, err) @@ -625,7 +625,7 @@ func TestRaceIntegration(t *testing.T) { theMetrics := pbsmetrics.NewMetrics(metrics.NewRegistry(), openrtb_ext.BidderList(), config.DisabledMetrics{}) currencyConverter := currencies.NewRateConverter(&http.Client{}, "", time.Duration(0)) ex := NewExchange(server.Client(), &wellBehavedCache{}, cfg, theMetrics, adapters.ParseBidderInfos(cfg.Adapters, "../static/bidder-info", openrtb_ext.BidderList()), gdpr.AlwaysAllow{}, currencyConverter) - _, err := ex.HoldAuction(context.Background(), newRaceCheckingRequest(t), &emptyUsersync{}, pbsmetrics.Labels{}, &categoriesFetcher, nil) + _, err := ex.HoldAuction(context.Background(), newRaceCheckingRequest(t), &emptyUsersync{}, pbsmetrics.Labels{}, &config.Account{}, &categoriesFetcher, nil) if err != nil { t.Errorf("HoldAuction returned unexpected error: %v", err) } @@ -812,7 +812,7 @@ func TestPanicRecoveryHighLevel(t *testing.T) { if error != nil { t.Errorf("Failed to create a category Fetcher: %v", error) } - _, err := e.HoldAuction(context.Background(), request, &emptyUsersync{}, pbsmetrics.Labels{}, &categoriesFetcher, nil) + _, err := e.HoldAuction(context.Background(), request, &emptyUsersync{}, pbsmetrics.Labels{}, &config.Account{}, &categoriesFetcher, nil) if err != nil { t.Errorf("HoldAuction returned unexpected error: %v", err) } @@ -928,7 +928,7 @@ func runSpec(t *testing.T, filename string, spec *exchangeSpec) { *debugLog = *spec.DebugLog debugLog.Regexp = regexp.MustCompile(`[<>]`) } - bid, err := ex.HoldAuction(context.Background(), &spec.IncomingRequest.OrtbRequest, mockIdFetcher(spec.IncomingRequest.Usersyncs), pbsmetrics.Labels{}, &categoriesFetcher, debugLog) + bid, err := ex.HoldAuction(context.Background(), &spec.IncomingRequest.OrtbRequest, mockIdFetcher(spec.IncomingRequest.Usersyncs), pbsmetrics.Labels{}, &config.Account{}, &categoriesFetcher, debugLog) responseTimes := extractResponseTimes(t, filename, bid) for _, bidderName := range biddersInAuction { if _, ok := responseTimes[bidderName]; !ok { diff --git a/exchange/targeting_test.go b/exchange/targeting_test.go index e596e5aa215..aaa75411ee4 100644 --- a/exchange/targeting_test.go +++ b/exchange/targeting_test.go @@ -108,7 +108,7 @@ func runTargetingAuction(t *testing.T, mockBids map[openrtb_ext.BidderName][]*op if error != nil { t.Errorf("Failed to create a category Fetcher: %v", error) } - bidResp, err := ex.HoldAuction(context.Background(), req, &mockFetcher{}, pbsmetrics.Labels{}, &categoriesFetcher, nil) + bidResp, err := ex.HoldAuction(context.Background(), req, &mockFetcher{}, pbsmetrics.Labels{}, &config.Account{}, &categoriesFetcher, nil) if err != nil { t.Fatalf("Unexpected errors running auction: %v", err) diff --git a/router/router.go b/router/router.go index 30936705a22..4826f0e5feb 100644 --- a/router/router.go +++ b/router/router.go @@ -211,7 +211,7 @@ func New(cfg *config.Configuration, rateConvertor *currencies.RateConverter) (r // Metrics engine r.MetricsEngine = metricsConf.NewMetricsEngine(cfg, legacyBidderList) - db, shutdown, fetcher, ampFetcher, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) + db, shutdown, fetcher, ampFetcher, accounts, categoriesFetcher, videoFetcher := storedRequestsConf.NewStoredRequests(cfg, r.MetricsEngine, generalHttpClient, r.Router) // todo(zachbadgett): better shutdown r.Shutdown = shutdown @@ -243,19 +243,19 @@ func New(cfg *config.Configuration, rateConvertor *currencies.RateConverter) (r cacheClient := pbc.NewClient(cacheHttpClient, &cfg.CacheURL, &cfg.ExtCacheURL, r.MetricsEngine) theExchange := exchange.NewExchange(generalHttpClient, cacheClient, cfg, r.MetricsEngine, bidderInfos, gdprPerms, rateConvertor) - openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator, fetcher, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap) + openrtbEndpoint, err := openrtb2.NewEndpoint(theExchange, paramsValidator, fetcher, accounts, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap) if err != nil { glog.Fatalf("Failed to create the openrtb endpoint handler. %v", err) } - ampEndpoint, err := openrtb2.NewAmpEndpoint(theExchange, paramsValidator, ampFetcher, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap) + ampEndpoint, err := openrtb2.NewAmpEndpoint(theExchange, paramsValidator, ampFetcher, accounts, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap) if err != nil { glog.Fatalf("Failed to create the amp endpoint handler. %v", err) } - videoEndpoint, err := openrtb2.NewVideoEndpoint(theExchange, paramsValidator, fetcher, videoFetcher, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap, cacheClient) + videoEndpoint, err := openrtb2.NewVideoEndpoint(theExchange, paramsValidator, fetcher, videoFetcher, accounts, categoriesFetcher, cfg, r.MetricsEngine, pbsAnalytics, disabledBidders, defReqJSON, activeBiddersMap, cacheClient) if err != nil { glog.Fatalf("Failed to create the video endpoint handler. %v", err) } diff --git a/stored_requests/backends/db_fetcher/fetcher.go b/stored_requests/backends/db_fetcher/fetcher.go index 223067c917e..541af835e82 100644 --- a/stored_requests/backends/db_fetcher/fetcher.go +++ b/stored_requests/backends/db_fetcher/fetcher.go @@ -8,6 +8,7 @@ import ( "github.com/lib/pq" "github.com/golang/glog" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/stored_requests" ) @@ -93,6 +94,10 @@ func (fetcher *dbFetcher) FetchRequests(ctx context.Context, requestIDs []string return storedRequestData, storedImpData, errs } +func (fetcher *dbFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + return nil, nil +} + func (fetcher *dbFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/backends/empty_fetcher/fetcher.go b/stored_requests/backends/empty_fetcher/fetcher.go index 25e8ead434b..b7201447a58 100644 --- a/stored_requests/backends/empty_fetcher/fetcher.go +++ b/stored_requests/backends/empty_fetcher/fetcher.go @@ -3,6 +3,8 @@ package empty_fetcher import ( "context" "encoding/json" + + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/stored_requests" ) @@ -27,6 +29,10 @@ func (fetcher EmptyFetcher) FetchRequests(ctx context.Context, requestIDs []stri return } +func (fetcher EmptyFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + return nil, nil +} + func (fetcher EmptyFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/backends/file_fetcher/fetcher.go b/stored_requests/backends/file_fetcher/fetcher.go index 60853f65da7..31b36e413fb 100644 --- a/stored_requests/backends/file_fetcher/fetcher.go +++ b/stored_requests/backends/file_fetcher/fetcher.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "strings" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/stored_requests" ) @@ -33,6 +34,24 @@ func (fetcher *eagerFetcher) FetchRequests(ctx context.Context, requestIDs []str return storedRequests, storedImpressions, errs } +// FetchAccount fetches the host account configuration for a publisher +func (fetcher *eagerFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + if len(accountID) == 0 { + return nil, fmt.Errorf("Cannot look up an empty accountID") + } + account := new(config.Account) + var err error + if rawJSON, ok := fetcher.FileSystem.Directories["accounts"].Files[accountID]; ok { + err = json.Unmarshal(rawJSON, account) + } else { + err = fmt.Errorf("Account %s not found", accountID) + } + if err != nil { + account = nil + } + return account, err +} + func (fetcher *eagerFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { fileName := primaryAdServer diff --git a/stored_requests/backends/http_fetcher/fetcher.go b/stored_requests/backends/http_fetcher/fetcher.go index b7e42c9e6cf..298ebb877ae 100644 --- a/stored_requests/backends/http_fetcher/fetcher.go +++ b/stored_requests/backends/http_fetcher/fetcher.go @@ -9,6 +9,7 @@ import ( "net/http" "strings" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/stored_requests" "github.com/golang/glog" @@ -81,6 +82,10 @@ func (fetcher *HttpFetcher) FetchRequests(ctx context.Context, requestIDs []stri return } +func (fetcher *HttpFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + return nil, nil +} + func (fetcher *HttpFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { if fetcher.Categories == nil { fetcher.Categories = make(map[string]map[string]stored_requests.Category) diff --git a/stored_requests/config/config.go b/stored_requests/config/config.go index e8f6b0bdf46..af690ff9345 100644 --- a/stored_requests/config/config.go +++ b/stored_requests/config/config.go @@ -105,7 +105,7 @@ func CreateStoredRequests(cfg *config.StoredRequestsSlim, metricsEngine pbsmetri // // As a side-effect, it will add some endpoints to the router if the config calls for it. // In the future we should look for ways to simplify this so that it's not doing two things. -func NewStoredRequests(cfg *config.Configuration, metricsEngine pbsmetrics.MetricsEngine, client *http.Client, router *httprouter.Router) (db *sql.DB, shutdown func(), fetcher stored_requests.Fetcher, ampFetcher stored_requests.Fetcher, categoriesFetcher stored_requests.CategoryFetcher, videoFetcher stored_requests.Fetcher) { +func NewStoredRequests(cfg *config.Configuration, metricsEngine pbsmetrics.MetricsEngine, client *http.Client, router *httprouter.Router) (db *sql.DB, shutdown func(), fetcher stored_requests.Fetcher, ampFetcher stored_requests.Fetcher, accountsFetcher stored_requests.AccountFetcher, categoriesFetcher stored_requests.CategoryFetcher, videoFetcher stored_requests.Fetcher) { // Build individual slim options from combined config struct slimAuction, slimAmp := resolvedStoredRequestsConfig(cfg) @@ -120,6 +120,7 @@ func NewStoredRequests(cfg *config.Configuration, metricsEngine pbsmetrics.Metri fetcher2, shutdown2 := CreateStoredRequests(&slimAmp, metricsEngine, client, router, &dbc) fetcher3, shutdown3 := CreateStoredRequests(&cfg.CategoryMapping, metricsEngine, client, router, &dbc) fetcher4, shutdown4 := CreateStoredRequests(&cfg.StoredVideo, metricsEngine, client, router, &dbc) + fetcher5, shutdown5 := CreateStoredRequests(&cfg.Accounts, metricsEngine, client, router, &dbc) db = dbc.db @@ -127,12 +128,14 @@ func NewStoredRequests(cfg *config.Configuration, metricsEngine pbsmetrics.Metri ampFetcher = fetcher2.(stored_requests.Fetcher) categoriesFetcher = fetcher3.(stored_requests.CategoryFetcher) videoFetcher = fetcher4.(stored_requests.Fetcher) + accountsFetcher = fetcher5.(stored_requests.AccountFetcher) shutdown = func() { shutdown1() shutdown2() shutdown3() shutdown4() + shutdown5() } return diff --git a/stored_requests/data/by_id/accounts/test.json b/stored_requests/data/by_id/accounts/test.json new file mode 100644 index 00000000000..76bafff7f1c --- /dev/null +++ b/stored_requests/data/by_id/accounts/test.json @@ -0,0 +1,14 @@ +{ + "id": "test", + "name": "test account", + "disabled": true, + "cache_ttl": { + "banner": 600, + "video": 3600, + "native": 3600, + "audio": 3600 + }, + "events": { + "enabled": true + } +} diff --git a/stored_requests/fetcher.go b/stored_requests/fetcher.go index 23fdb6b4925..a8d08f11ad4 100644 --- a/stored_requests/fetcher.go +++ b/stored_requests/fetcher.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/pbsmetrics" ) @@ -25,6 +26,11 @@ type Fetcher interface { FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) } +type AccountFetcher interface { + // FetchAccount fetches the host account configuration for a publisher + FetchAccount(ctx context.Context, accountID string) (*config.Account, error) +} + type CategoryFetcher interface { // FetchCategories fetches the ad-server/publisher specific category for the given IAB category FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) @@ -33,6 +39,7 @@ type CategoryFetcher interface { // AllFetcher is an interface that encapsulates both the original Fetcher and the CategoryFetcher type AllFetcher interface { FetchRequests(ctx context.Context, requestIDs []string, impIDs []string) (requestData map[string]json.RawMessage, impData map[string]json.RawMessage, errs []error) + FetchAccount(ctx context.Context, accountID string) (*config.Account, error) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) } @@ -181,6 +188,10 @@ func (f *fetcherWithCache) FetchRequests(ctx context.Context, requestIDs []strin return } +func (f *fetcherWithCache) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + return f.fetcher.FetchAccount(ctx, accountID) +} + func (f *fetcherWithCache) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/fetcher_test.go b/stored_requests/fetcher_test.go index c1040acdb90..ec41e5e644f 100644 --- a/stored_requests/fetcher_test.go +++ b/stored_requests/fetcher_test.go @@ -6,6 +6,7 @@ import ( "errors" "testing" + "github.com/prebid/prebid-server/config" "github.com/prebid/prebid-server/pbsmetrics" "github.com/stretchr/testify/assert" @@ -215,6 +216,10 @@ func (f *mockFetcher) FetchRequests(ctx context.Context, requestIDs []string, im return args.Get(0).(map[string]json.RawMessage), args.Get(1).(map[string]json.RawMessage), args.Get(2).([]error) } +func (a *mockFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + return nil, nil +} + func (f *mockFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { return "", nil } diff --git a/stored_requests/multifetcher.go b/stored_requests/multifetcher.go index 24cf848448c..39bd921030d 100644 --- a/stored_requests/multifetcher.go +++ b/stored_requests/multifetcher.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + + "github.com/prebid/prebid-server/config" ) // MultiFetcher is a Fetcher composed of multiple sub-Fetchers that are all polled for results. @@ -36,6 +38,17 @@ func (mf MultiFetcher) FetchRequests(ctx context.Context, requestIDs []string, i return } +func (mf MultiFetcher) FetchAccount(ctx context.Context, accountID string) (*config.Account, error) { + for _, f := range mf { + if af, ok := f.(AccountFetcher); ok { + if account, err := af.FetchAccount(ctx, accountID); err == nil { + return account, nil + } + } + } + return nil, NotFoundError{accountID, "Account"} +} + func (mf MultiFetcher) FetchCategories(ctx context.Context, primaryAdServer, publisherId, iabCategory string) (string, error) { for _, f := range mf { if cf, ok := f.(CategoryFetcher); ok {