Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix pyfunc bugs for release #188

Merged
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
29c6a18
Refactor pyfunc ensemble method to use input as arg name
deadlycoconuts Mar 24, 2022
e86f394
Hide the search bar for ensemblers temporarily
deadlycoconuts Mar 24, 2022
d59f759
Refactor predictions in pyfunc interface
deadlycoconuts Mar 24, 2022
4655216
Refactor predictions in pyfunc ensembler sample
deadlycoconuts Mar 25, 2022
d3244b4
Fix renaming of features to input
deadlycoconuts Mar 25, 2022
ccabb46
Refactor pyfunc ensembler service engine
deadlycoconuts Mar 25, 2022
b02edc9
Update SDK version number
deadlycoconuts Mar 25, 2022
4293ec4
Fix undeterministic experiment behaviour for e2e tests
deadlycoconuts Mar 25, 2022
f41ce6b
Refactor name change from features to input in batch ensembling engine
deadlycoconuts Mar 25, 2022
1ec08af
Add ensembler name search to API
deadlycoconuts Mar 25, 2022
6b4bc48
Update openapi generated ensembler class
deadlycoconuts Mar 25, 2022
8720668
Add description to search query for list ensemblers endpoint
deadlycoconuts Mar 25, 2022
471f6d5
Revert version number to base number
deadlycoconuts Mar 25, 2022
a5063ec
Replace ensembler/job name search to be case-insensitive
deadlycoconuts Mar 25, 2022
c3ac01f
Remove random tag created
deadlycoconuts Mar 25, 2022
353954a
Revert infra values to original
deadlycoconuts Mar 28, 2022
df35fd0
Drop python versions for ensembler engines
deadlycoconuts Mar 28, 2022
4693f44
Fix jinja2 version for batch ensembler engine
deadlycoconuts Mar 28, 2022
c47112b
Simplify API description for jobs name query string
deadlycoconuts Mar 28, 2022
481c693
Drop ensembler engines setup files Python version to 3.7
deadlycoconuts Mar 28, 2022
5c053b2
- fix treatments assigner
romanwozniak Mar 28, 2022
808818c
- sort exp treatments by the traffic %
romanwozniak Mar 28, 2022
99fdb28
Fix ensembler engines and SDK to all use Python 3.7.*
deadlycoconuts Mar 28, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion engines/experiment/examples/plugins/hardcoded/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ type SegmenterConfig struct {

type Experiment struct {
manager.Experiment
SegmentationConfig SegmenterConfig
SegmentationConfig SegmenterConfig `json:"segmentation_configuration"`
VariantsConfig map[string]TreatmentConfig `json:"variants_configuration"`
}

Expand Down
25 changes: 21 additions & 4 deletions engines/experiment/examples/plugins/hardcoded/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,12 @@ import (
"github.com/gojek/turing/engines/experiment/pkg/request"
"github.com/gojek/turing/engines/experiment/runner"
"net/http"
"sort"
)

type ExperimentRunner struct {
experiments []Experiment
experiments []Experiment
sortedTreatments map[string][]string
}

func (e *ExperimentRunner) Configure(cfg json.RawMessage) error {
Expand All @@ -19,8 +21,22 @@ func (e *ExperimentRunner) Configure(cfg json.RawMessage) error {
if err != nil {
return err
}

e.experiments = config.Experiments
e.sortedTreatments = make(map[string][]string)

for _, exp := range e.experiments {
var variants []string

for name := range exp.VariantsConfig {
variants = append(variants, name)
}

sort.Slice(variants, func(i, j int) bool {
return exp.VariantsConfig[variants[i]].Traffic > exp.VariantsConfig[variants[j]].Traffic
})

e.sortedTreatments[exp.ID] = variants
}
return nil
}

Expand All @@ -42,9 +58,10 @@ func (e *ExperimentRunner) GetTreatmentForRequest(
bucket := utils.Hash(segmentationUnit) % 10000

var total uint32 = 0
for name, variant := range exp.VariantsConfig {
for _, name := range e.sortedTreatments[exp.ID] {
variant := exp.VariantsConfig[name]
total += uint32(variant.Traffic * 10000)
if bucket < total {
if bucket <= total {
return &runner.Treatment{
ExperimentName: exp.Name,
Name: name,
Expand Down
86 changes: 41 additions & 45 deletions engines/experiment/examples/plugins/hardcoded/runner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,81 +5,77 @@ import (
"net/http"
"testing"

"github.com/gojek/turing/engines/experiment/manager"
"github.com/gojek/turing/engines/experiment/pkg/request"
"github.com/gojek/turing/engines/experiment/runner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestExperimentRunner_GetTreatmentForRequest(t *testing.T) {
experiments := []Experiment{
{
Experiment: manager.Experiment{
ID: "001",
Name: "exp_1",
Variants: []manager.Variant{
{
Name: "control",
},
{
Name: "treatment-1",
},
},
},
SegmentationConfig: SegmenterConfig{
Name: "customer_id",
SegmenterSource: request.PayloadFieldSource,
SegmenterValue: "client.id",
},
VariantsConfig: map[string]TreatmentConfig{
"control": {
Traffic: 0.85,
Data: json.RawMessage(`{"foo": "bar"}`),
},
"treatment-1": {
Traffic: 0.15,
Data: json.RawMessage(`{"bar": "baz"}`),
runnerConfig := json.RawMessage(`{
"experiments": [
{
"id": "001",
"name": "exp_1",
"variants": [
{"name": "control"},
{"name": "treatment-1"}
],
"segmentation_configuration": {
"name": "client_id",
"source": "payload",
"value": "client.id"
},
},
},
}
"variants_configuration": {
"control": {
"traffic": 0.85,
"treatment_configuration": {"foo": "bar"}
},
"treatment-1": {
"traffic": 0.15,
"treatment_configuration": {"bar": "baz"}
}
}
}
]
}`)

suite := map[string]struct {
experiments []Experiment
header http.Header
payload json.RawMessage
expected *runner.Treatment
err string
runnerConfig json.RawMessage
header http.Header
payload json.RawMessage
expected *runner.Treatment
err string
}{
"success | client_id:4": {
experiments: experiments,
payload: json.RawMessage(`{"client": {"id": 4}}`),
runnerConfig: runnerConfig,
payload: json.RawMessage(`{"client": {"id": 4}}`),
expected: &runner.Treatment{
ExperimentName: "exp_1",
Name: "control",
Config: json.RawMessage(`{"foo": "bar"}`),
},
},
"success | client_id:7": {
experiments: experiments,
payload: json.RawMessage(`{"client": {"id": 7}}`),
runnerConfig: runnerConfig,
payload: json.RawMessage(`{"client": {"id": 7}}`),
expected: &runner.Treatment{
ExperimentName: "exp_1",
Name: "treatment-1",
Config: json.RawMessage(`{"bar": "baz"}`),
},
},
"failure": {
experiments: experiments,
payload: json.RawMessage(`{}`),
err: "no experiment configured for the unit",
runnerConfig: runnerConfig,
payload: json.RawMessage(`{}`),
err: "no experiment configured for the unit",
},
}

for name, tt := range suite {
t.Run(name, func(t *testing.T) {
expRunner := ExperimentRunner{experiments: tt.experiments}
expRunner := &ExperimentRunner{}
err := expRunner.Configure(tt.runnerConfig)
require.NoError(t, err)

actual, err := expRunner.GetTreatmentForRequest(tt.header, tt.payload, runner.GetTreatmentOptions{})
if tt.err != "" {
Expand Down