Skip to content

Commit

Permalink
ip address geolocation lookup
Browse files Browse the repository at this point in the history
Signed-off-by: Kehan Pan <[email protected]>
  • Loading branch information
aptxx committed Sep 16, 2024
1 parent 640b97c commit 16b7183
Show file tree
Hide file tree
Showing 50 changed files with 4,072 additions and 113 deletions.
16 changes: 13 additions & 3 deletions config/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type Account struct {
DefaultBidLimit int `mapstructure:"default_bid_limit" json:"default_bid_limit"`
BidAdjustments *openrtb_ext.ExtRequestPrebidBidAdjustments `mapstructure:"bidadjustments" json:"bidadjustments"`
Privacy AccountPrivacy `mapstructure:"privacy" json:"privacy"`
GeoLocation AccountGeoLocation `mapstructure:"geolocation" json:"geolocation"`
}

// CookieSync represents the account-level defaults for the cookie sync endpoint.
Expand Down Expand Up @@ -156,9 +157,10 @@ type AccountGDPR struct {
Purpose9 AccountGDPRPurpose `mapstructure:"purpose9" json:"purpose9"`
Purpose10 AccountGDPRPurpose `mapstructure:"purpose10" json:"purpose10"`
// Hash table of purpose configs for convenient purpose config lookup
PurposeConfigs map[consentconstants.Purpose]*AccountGDPRPurpose
PurposeOneTreatment AccountGDPRPurposeOneTreatment `mapstructure:"purpose_one_treatment" json:"purpose_one_treatment"`
SpecialFeature1 AccountGDPRSpecialFeature `mapstructure:"special_feature1" json:"special_feature1"`
PurposeConfigs map[consentconstants.Purpose]*AccountGDPRPurpose
PurposeOneTreatment AccountGDPRPurposeOneTreatment `mapstructure:"purpose_one_treatment" json:"purpose_one_treatment"`
SpecialFeature1 AccountGDPRSpecialFeature `mapstructure:"special_feature1" json:"special_feature1"`
ConsentStringMeansInScope *bool `mapstructure:"consent_string_means_in_scope" json:"consent_string_means_in_scope"`
}

// EnabledForChannelType indicates whether GDPR is turned on at the account level for the specified channel type
Expand Down Expand Up @@ -351,6 +353,14 @@ type CookieDeprecation struct {
TTLSec int `mapstructure:"ttl_sec"`
}

type AccountGeoLocation struct {
Enabled bool `mapstructure:"enabled" json:"enabled,omitempty"`
}

func (g *AccountGeoLocation) IsGeoLocationEnabled() bool {
return g.Enabled
}

// AccountDSA represents DSA configuration
type AccountDSA struct {
Default string `mapstructure:"default" json:"default"`
Expand Down
24 changes: 24 additions & 0 deletions config/account_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1014,3 +1014,27 @@ func TestIPMaskingValidate(t *testing.T) {
})
}
}

func TestGeoLocation(t *testing.T) {
tests := []struct {
geoloc *AccountGeoLocation
expected bool
}{
{
geoloc: &AccountGeoLocation{
Enabled: true,
},
expected: true,
},
{
geoloc: &AccountGeoLocation{
Enabled: false,
},
expected: false,
},
}

for _, test := range tests {
assert.Equal(t, test.expected, test.geoloc.IsGeoLocationEnabled())
}
}
43 changes: 41 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ type Configuration struct {
Hooks Hooks `mapstructure:"hooks"`
Validations Validations `mapstructure:"validations"`
PriceFloors PriceFloors `mapstructure:"price_floors"`
GeoLocation GeoLocation `mapstructure:"geolocation"`
}

type Admin struct {
Expand Down Expand Up @@ -253,8 +254,9 @@ type GDPR struct {
// If the gdpr flag is unset in a request, but geo.country is set, we will assume GDPR applies if and only
// if the country matches one on this list. If both the GDPR flag and country are not set, we default
// to DefaultValue
EEACountries []string `mapstructure:"eea_countries"`
EEACountriesMap map[string]struct{}
EEACountries []string `mapstructure:"eea_countries"`
EEACountriesMap map[string]struct{}
ConsentStringMeansInScope bool `mapstructure:"consent_string_means_in_scope"`
}

func (cfg *GDPR) validate(v *viper.Viper, errs []error) []error {
Expand Down Expand Up @@ -659,6 +661,27 @@ type DefReqFiles struct {
FileName string `mapstructure:"name"`
}

type GeoLocation struct {
Enabled bool `mapstructure:"enabled"`
Type string `mapstructure:"type"`
Maxmind GeoLocationMaxmind `mapstructure:"maxmind"`
}

type GeoLocationMaxmind struct {
RemoteFileSyncer MaxmindRemoteFileSyncer `mapstructure:"remote_file_syncer"`
}

type MaxmindRemoteFileSyncer struct {
HttpClient HTTPClient `mapstructure:"http_client"`
DownloadURL string `mapstructure:"download_url"`
SaveFilePath string `mapstructure:"save_filepath"`
TmpFilePath string `mapstructure:"tmp_filepath"`
RetryCount int `mapstructure:"retry_count"`
RetryIntervalMillis int `mapstructure:"retry_interval_ms"`
TimeoutMillis int `mapstructure:"timeout_ms"`
UpdateIntervalMillis int `mapstructure:"update_interval_ms"`
}

type Debug struct {
TimeoutNotification TimeoutNotification `mapstructure:"timeout_notification"`
OverrideToken string `mapstructure:"override_token"`
Expand Down Expand Up @@ -1140,6 +1163,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
"FIN", "FRA", "GUF", "DEU", "GIB", "GRC", "GLP", "GGY", "HUN", "ISL", "IRL", "IMN", "ITA", "JEY", "LVA",
"LIE", "LTU", "LUX", "MLT", "MTQ", "MYT", "NLD", "NOR", "POL", "PRT", "REU", "ROU", "BLM", "MAF", "SPM",
"SVK", "SVN", "ESP", "SWE", "GBR"})
v.SetDefault("gdpr.consent_string_means_in_scope", false)
v.SetDefault("ccpa.enforce", false)
v.SetDefault("lmt.enforce", true)
v.SetDefault("currency_converter.fetch_url", "https://cdn.jsdelivr.net/gh/prebid/currency-file@1/latest.json")
Expand Down Expand Up @@ -1171,6 +1195,7 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
v.SetDefault("account_defaults.privacy.privacysandbox.topicsdomain", "")
v.SetDefault("account_defaults.privacy.privacysandbox.cookiedeprecation.enabled", false)
v.SetDefault("account_defaults.privacy.privacysandbox.cookiedeprecation.ttl_sec", 604800)
v.SetDefault("account_defaults.geolocation.enabled", false)

v.SetDefault("account_defaults.events_enabled", false)
v.BindEnv("account_defaults.privacy.dsa.default")
Expand All @@ -1188,6 +1213,20 @@ func SetupViper(v *viper.Viper, filename string, bidderInfos BidderInfos) {
v.SetDefault("price_floors.fetcher.http_client.idle_connection_timeout_seconds", 60)
v.SetDefault("price_floors.fetcher.max_retries", 10)

v.SetDefault("geolocation.enabled", false)
v.SetDefault("geolocation.type", "maxmind")
v.SetDefault("geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz")
v.SetDefault("geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz")
v.SetDefault("geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz")
v.SetDefault("geolocation.maxmind.remote_file_syncer.retry_count", 3)
v.SetDefault("geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000)
v.SetDefault("geolocation.maxmind.remote_file_syncer.timeout_ms", 300000)
v.SetDefault("geolocation.maxmind.remote_file_syncer.update_interval_ms", 0)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2)
v.SetDefault("geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60)

v.SetDefault("account_defaults.events_enabled", false)
v.SetDefault("compression.response.enable_gzip", false)
v.SetDefault("compression.request.enable_gzip", false)
Expand Down
53 changes: 53 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,24 @@ func TestDefaults(t *testing.T) {
cmpStrings(t, "account_defaults.privacy.topicsdomain", "", cfg.AccountDefaults.Privacy.PrivacySandbox.TopicsDomain)
cmpBools(t, "account_defaults.privacy.privacysandbox.cookiedeprecation.enabled", false, cfg.AccountDefaults.Privacy.PrivacySandbox.CookieDeprecation.Enabled)
cmpInts(t, "account_defaults.privacy.privacysandbox.cookiedeprecation.ttl_sec", 604800, cfg.AccountDefaults.Privacy.PrivacySandbox.CookieDeprecation.TTLSec)
cmpBools(t, "account_defaults.geolocation.enabled", false, cfg.AccountDefaults.GeoLocation.Enabled)

cmpBools(t, "account_defaults.events.enabled", false, cfg.AccountDefaults.Events.Enabled)

cmpBools(t, "geolocation.enabled", false, cfg.GeoLocation.Enabled)
cmpStrings(t, "geolocation.type", "maxmind", cfg.GeoLocation.Type)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_count", 3, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.timeout_ms", 300000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.update_interval_ms", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis)

cmpBools(t, "hooks.enabled", false, cfg.Hooks.Enabled)
cmpStrings(t, "validations.banner_creative_max_size", "skip", cfg.Validations.BannerCreativeMaxSize)
cmpStrings(t, "validations.secure_markup", "skip", cfg.Validations.SecureMarkup)
Expand All @@ -227,6 +242,8 @@ func TestDefaults(t *testing.T) {
cmpInts(t, "account_defaults.privacy.ipv6.anon_keep_bits", 56, cfg.AccountDefaults.Privacy.IPv6Config.AnonKeepBits)
cmpInts(t, "account_defaults.privacy.ipv4.anon_keep_bits", 24, cfg.AccountDefaults.Privacy.IPv4Config.AnonKeepBits)

cmpBools(t, "gdpr.consent_string_means_in_scope", false, cfg.GDPR.ConsentStringMeansInScope)

//Assert purpose VendorExceptionMap hash tables were built correctly
cmpBools(t, "analytics.agma.enabled", false, cfg.Analytics.Agma.Enabled)
cmpStrings(t, "analytics.agma.endpoint.timeout", "2s", cfg.Analytics.Agma.Endpoint.Timeout)
Expand Down Expand Up @@ -350,6 +367,7 @@ gdpr:
default_value: "1"
non_standard_publishers: ["pub1", "pub2"]
eea_countries: ["eea1", "eea2"]
consent_string_means_in_scope: false
tcf2:
purpose1:
enforce_vendors: false
Expand Down Expand Up @@ -494,6 +512,23 @@ price_floors:
max_idle_connections_per_host: 2
idle_connection_timeout_seconds: 10
max_retries: 5
geolocation:
enabled: false
type: maxmind
maxmind:
remote_file_syncer:
http_client:
max_connections_per_host: 0
max_idle_connections: 40
max_idle_connections_per_host: 2
idle_connection_timeout_seconds: 60
download_url: "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"
save_filepath: "/var/tmp/prebid/GeoLite2-City.tar.gz"
tmp_filepath: "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz"
retry_count: 3
retry_interval_ms: 3000
timeout_ms: 300000
update_interval_ms: 0
account_defaults:
events:
enabled: true
Expand Down Expand Up @@ -541,6 +576,8 @@ account_defaults:
cookiedeprecation:
enabled: true
ttl_sec: 86400
geolocation:
enabled: false
tmax_adjustments:
enabled: true
bidder_response_duration_min_ms: 700
Expand Down Expand Up @@ -631,6 +668,7 @@ func TestFullConfig(t *testing.T) {
cmpInts(t, "http_client_cache.idle_connection_timeout_seconds", 3, cfg.CacheClient.IdleConnTimeout)
cmpInts(t, "gdpr.host_vendor_id", 15, cfg.GDPR.HostVendorID)
cmpStrings(t, "gdpr.default_value", "1", cfg.GDPR.DefaultValue)
cmpBools(t, "gdpr.consent_string_means_in_scope", false, cfg.GDPR.ConsentStringMeansInScope)
cmpStrings(t, "host_schain_node.asi", "pbshostcompany.com", cfg.HostSChainNode.ASI)
cmpStrings(t, "host_schain_node.sid", "00001", cfg.HostSChainNode.SID)
cmpStrings(t, "host_schain_node.rid", "BidRequest", cfg.HostSChainNode.RID)
Expand Down Expand Up @@ -890,6 +928,21 @@ func TestFullConfig(t *testing.T) {
cmpStrings(t, "analytics.agma.accounts.0.publisher_id", "publisher-id", cfg.Analytics.Agma.Accounts[0].PublisherId)
cmpStrings(t, "analytics.agma.accounts.0.code", "agma-code", cfg.Analytics.Agma.Accounts[0].Code)
cmpStrings(t, "analytics.agma.accounts.0.site_app_id", "site-or-app-id", cfg.Analytics.Agma.Accounts[0].SiteAppId)

cmpBools(t, "account_defaults.geolocation.enabled", false, cfg.GeoLocation.Enabled)
cmpBools(t, "geolocation.enabled", false, cfg.GeoLocation.Enabled)
cmpStrings(t, "geolocation.type", "maxmind", cfg.GeoLocation.Type)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_connections_per_host", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections", 40, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConns)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.max_idle_connections_per_host", 2, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.MaxIdleConnsPerHost)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.http_client.idle_connection_timeout_seconds", 60, cfg.GeoLocation.Maxmind.RemoteFileSyncer.HttpClient.IdleConnTimeout)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.download_url", "https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.DownloadURL)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.save_filepath", "/var/tmp/prebid/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.SaveFilePath)
cmpStrings(t, "geolocation.maxmind.remote_file_syncer.tmp_filepath", "/var/tmp/prebid/tmp/GeoLite2-City.tar.gz", cfg.GeoLocation.Maxmind.RemoteFileSyncer.TmpFilePath)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_count", 3, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryCount)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.retry_interval_ms", 3000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.RetryIntervalMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.timeout_ms", 300000, cfg.GeoLocation.Maxmind.RemoteFileSyncer.TimeoutMillis)
cmpInts(t, "geolocation.maxmind.remote_file_syncer.update_interval_ms", 0, cfg.GeoLocation.Maxmind.RemoteFileSyncer.UpdateIntervalMillis)
}

func TestValidateConfig(t *testing.T) {
Expand Down
63 changes: 63 additions & 0 deletions config/countrycode/countrycode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
package countrycode

import (
"strings"
)

type CountryCode struct {
map2To3 map[string]string
map3To2 map[string]string
}

func New() *CountryCode {
return &CountryCode{
map2To3: make(map[string]string),
map3To2: make(map[string]string),
}
}

// Load loads country code mapping data
func (c *CountryCode) Load(data string) {
toAlpha2 := make(map[string]string)
toAlpha3 := make(map[string]string)
for _, line := range strings.Split(data, "\n") {
if line == "" {
continue
}
fields := strings.Split(line, ",")
if len(fields) < 2 {
continue
}
alpha2 := strings.TrimSpace(fields[0])
alpha3 := strings.TrimSpace(fields[1])
toAlpha2[alpha3] = alpha2
toAlpha3[alpha2] = alpha3
}

c.map2To3 = toAlpha3
c.map3To2 = toAlpha2
}

// ToAlpha3 converts country code alpha2 to alpha3
func (c *CountryCode) ToAlpha3(alpha2 string) string {
return c.map2To3[alpha2]
}

// ToAlpha2 converts country code alpha3 to alpha2
func (c *CountryCode) ToAlpha2(alpha3 string) string {
return c.map3To2[alpha3]
}

var defaultCountryCode = New()

func Load(data string) {
defaultCountryCode.Load(data)
}

func ToAlpha3(alpha2 string) string {
return defaultCountryCode.ToAlpha3(alpha2)
}

func ToAlpha2(alpha3 string) string {
return defaultCountryCode.ToAlpha2(alpha3)
}
60 changes: 60 additions & 0 deletions config/countrycode/countrycode_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
package countrycode

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestCountryCode(t *testing.T) {
Load(`
AD,AND
AE,ARE
AF,AFG
`)
assert.Equal(t, "AND", ToAlpha3("AD"), "map AD to AND")
assert.Equal(t, "AE", ToAlpha2("ARE"), "map ARE to AE")
}

func TestCountryCodeToAlpha3(t *testing.T) {
c := New()
c.Load(`
AD,AND
AE,ARE
AF,AFG
`)
tests := []struct {
input string
expected string
}{
{"AD", "AND"},
{"AE", "ARE"},
{"XX", ""},
{"", ""},
}

for _, test := range tests {
assert.Equal(t, test.expected, c.ToAlpha3(test.input), "map %s to alpha3", test.input)
}
}

func TestCountryCodeToAlpha2(t *testing.T) {
c := New()
c.Load(`
AD,AND
AE,ARE
AF,AFG
`)
tests := []struct {
input string
expected string
}{
{"AND", "AD"},
{"ARE", "AE"},
{"", ""},
}

for _, test := range tests {
assert.Equal(t, test.expected, c.ToAlpha2(test.input), "map %s to alpha2", test.input)
}
}
Loading

0 comments on commit 16b7183

Please sign in to comment.