diff --git a/exporter/feature_event.go b/exporter/feature_event.go index 1fdfa02a822..ed77a3464f0 100644 --- a/exporter/feature_event.go +++ b/exporter/feature_event.go @@ -1,6 +1,7 @@ package exporter import ( + "encoding/json" "time" "github.com/thomaspoignant/go-feature-flag/ffuser" @@ -33,38 +34,52 @@ func NewFeatureEvent( } // FeatureEvent represent an event that we store in the data storage +// nolint:lll type FeatureEvent struct { // Kind for a feature event is feature. // A feature event will only be generated if the trackEvents attribute of the flag is set to true. - Kind string `json:"kind" example:"feature"` + Kind string `json:"kind" example:"feature" parquet:"name=kind, type=BYTE_ARRAY, convertedtype=UTF8"` // ContextKind is the kind of context which generated an event. This will only be "anonymousUser" for events generated // on behalf of an anonymous user or the reserved word "user" for events generated on behalf of a non-anonymous user - ContextKind string `json:"contextKind,omitempty" example:"user"` + ContextKind string `json:"contextKind,omitempty" example:"user" parquet:"name=contextKind, type=BYTE_ARRAY, convertedtype=UTF8"` // UserKey The key of the user object used in a feature flag evaluation. Details for the user object used in a feature // flag evaluation as reported by the "feature" event are transmitted periodically with a separate index event. - UserKey string `json:"userKey" example:"94a25909-20d8-40cc-8500-fee99b569345"` + UserKey string `json:"userKey" example:"94a25909-20d8-40cc-8500-fee99b569345" parquet:"name=userKey, type=BYTE_ARRAY, convertedtype=UTF8"` // CreationDate When the feature flag was requested at Unix epoch time in milliseconds. - CreationDate int64 `json:"creationDate" example:"1680246000011"` + CreationDate int64 `json:"creationDate" example:"1680246000011" parquet:"name=creationDate, type=INT64"` // Key of the feature flag requested. - Key string `json:"key" example:"my-feature-flag"` + Key string `json:"key" example:"my-feature-flag" parquet:"name=key, type=BYTE_ARRAY, convertedtype=UTF8"` // Variation of the flag requested. Flag variation values can be "True", "False", "Default" or "SdkDefault" // depending on which value was taken during flag evaluation. "SdkDefault" is used when an error is detected and the // default value passed during the call to your variation is used. - Variation string `json:"variation" example:"admin-variation"` + Variation string `json:"variation" example:"admin-variation" parquet:"name=variation, type=BYTE_ARRAY, convertedtype=UTF8"` // Value of the feature flag returned by feature flag evaluation. - Value interface{} `json:"value"` + Value interface{} `json:"value" parquet:"name=value, type=BYTE_ARRAY, convertedtype=UTF8"` // Default value is set to true if feature flag evaluation failed, in which case the value returned was the default // value passed to variation. If the default field is omitted, it is assumed to be false. - Default bool `json:"default" example:"false"` + Default bool `json:"default" example:"false" parquet:"name=default, type=BOOLEAN"` // Version contains the version of the flag. If the field is omitted for the flag in the configuration file // the default version will be 0. - Version string `json:"version" example:"v1.0.0"` + Version string `json:"version" example:"v1.0.0" parquet:"name=version, type=BYTE_ARRAY, convertedtype=UTF8"` +} + +// MarshalInterface marshals all interface type fields in FeatureEvent into JSON-encoded string. +func (f *FeatureEvent) MarshalInterface() error { + if f == nil { + return nil + } + b, err := json.Marshal(f.Value) + if err != nil { + return err + } + f.Value = string(b) + return nil } diff --git a/exporter/feature_event_test.go b/exporter/feature_event_test.go index a51e54664f6..fa2b010b4e4 100644 --- a/exporter/feature_event_test.go +++ b/exporter/feature_event_test.go @@ -45,3 +45,70 @@ func TestNewFeatureEvent(t *testing.T) { }) } } + +func TestFeatureEvent_MarshalInterface(t *testing.T) { + tests := []struct { + name string + featureEvent *exporter.FeatureEvent + want *exporter.FeatureEvent + wantErr bool + }{ + { + name: "happy path", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + }, + want: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: `{"bool":true,"float":1.23,"int":1,"string":"string"}`, + Default: false, + }, + }, + { + name: "marshal failed", + featureEvent: &exporter.FeatureEvent{ + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: make(chan int), + Default: false, + }, + wantErr: true, + }, + { + name: "nil featureEvent", + featureEvent: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.featureEvent.MarshalInterface(); (err != nil) != tt.wantErr { + t.Errorf("FeatureEvent.MarshalInterface() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.want != nil { + assert.Equal(t, tt.want, tt.featureEvent) + } + }) + } +} diff --git a/exporter/fileexporter/exporter.go b/exporter/fileexporter/exporter.go index 73f371bb34d..973c44ad269 100644 --- a/exporter/fileexporter/exporter.go +++ b/exporter/fileexporter/exporter.go @@ -5,16 +5,20 @@ import ( "fmt" "log" "os" + "runtime" "strings" "sync" "text/template" "github.com/thomaspoignant/go-feature-flag/exporter" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/parquet" + "github.com/xitongsys/parquet-go/writer" ) type Exporter struct { // Format is the output format you want in your exported file. - // Available format are JSON and CSV. + // Available format are JSON, CSV, and Parquet. // Default: JSON Format string @@ -37,6 +41,11 @@ type Exporter struct { // {{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n CsvTemplate string + // ParquetCompressionCodec is the parquet compression codec for better space efficiency. + // Available options https://github.com/apache/parquet-format/blob/master/Compression.md + // Default: SNAPPY + ParquetCompressionCodec string + csvTemplate *template.Template filenameTemplate *template.Template initTemplates sync.Once @@ -54,6 +63,7 @@ func (f *Exporter) Export(_ context.Context, _ *log.Logger, featureEvents []expo if f.Format == "" { f.Format = "json" } + f.Format = strings.ToLower(f.Format) // Get the filename filename, err := exporter.ComputeFilename(f.filenameTemplate, f.Format) @@ -63,6 +73,19 @@ func (f *Exporter) Export(_ context.Context, _ *log.Logger, featureEvents []expo filePath := f.OutputDir + "/" + filename + if f.Format == "parquet" { + return f.writeParquet(filePath, featureEvents) + } + return f.writeFile(filePath, featureEvents) +} + +// IsBulk return false if we should directly send the data as soon as it is produce +// and true if we collect the data to send them in bulk. +func (f *Exporter) IsBulk() bool { + return true +} + +func (f *Exporter) writeFile(filePath string, featureEvents []exporter.FeatureEvent) error { file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) if err != nil { return err @@ -73,7 +96,7 @@ func (f *Exporter) Export(_ context.Context, _ *log.Logger, featureEvents []expo var err error // Convert the line in the right format - switch strings.ToLower(f.Format) { + switch f.Format { case "csv": line, err = exporter.FormatEventInCSV(f.csvTemplate, event) case "json": @@ -94,8 +117,31 @@ func (f *Exporter) Export(_ context.Context, _ *log.Logger, featureEvents []expo return nil } -// IsBulk return false if we should directly send the data as soon as it is produce -// and true if we collect the data to send them in bulk. -func (f *Exporter) IsBulk() bool { - return true +func (f *Exporter) writeParquet(filePath string, featureEvents []exporter.FeatureEvent) error { + fw, err := local.NewLocalFileWriter(filePath) + if err != nil { + return err + } + defer fw.Close() + + pw, err := writer.NewParquetWriter(fw, new(exporter.FeatureEvent), int64(runtime.NumCPU())) + if err != nil { + return err + } + + pw.CompressionType = parquet.CompressionCodec_SNAPPY + if ct, err := parquet.CompressionCodecFromString(f.ParquetCompressionCodec); err == nil { + pw.CompressionType = ct + } + + for _, event := range featureEvents { + if err := event.MarshalInterface(); err != nil { + return err + } + if err = pw.Write(event); err != nil { + return fmt.Errorf("error while writing the export file: %v", err) + } + } + + return pw.WriteStop() } diff --git a/exporter/fileexporter/exporter_test.go b/exporter/fileexporter/exporter_test.go index 8f37e281e0b..4d9758dae96 100644 --- a/exporter/fileexporter/exporter_test.go +++ b/exporter/fileexporter/exporter_test.go @@ -4,22 +4,25 @@ import ( "context" "log" "os" + "runtime" "testing" - "github.com/thomaspoignant/go-feature-flag/exporter/fileexporter" - "github.com/stretchr/testify/assert" - "github.com/thomaspoignant/go-feature-flag/exporter" + "github.com/thomaspoignant/go-feature-flag/exporter/fileexporter" + "github.com/xitongsys/parquet-go-source/local" + "github.com/xitongsys/parquet-go/parquet" + "github.com/xitongsys/parquet-go/reader" ) func TestFile_Export(t *testing.T) { hostname, _ := os.Hostname() type fields struct { - Format string - Filename string - CsvTemplate string - OutputDir string + Format string + Filename string + CsvTemplate string + OutputDir string + ParquetCompressionCodec string } type args struct { logger *log.Logger @@ -28,6 +31,7 @@ func TestFile_Export(t *testing.T) { type expected struct { fileNameRegex string content string + featureEvents []exporter.FeatureEvent } tests := []struct { name string @@ -80,6 +84,39 @@ func TestFile_Export(t *testing.T) { content: "./testdata/all_default.csv", }, }, + { + name: "all default parquet", + wantErr: false, + fields: fields{ + Format: "parquet", + ParquetCompressionCodec: parquet.CompressionCodec_SNAPPY.String(), + }, + args: args{ + featureEvents: []exporter.FeatureEvent{ + { + Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", + Variation: "Default", Value: "YO", Default: false, + }, + { + Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", + Variation: "Default", Value: "YO2", Default: false, Version: "127", + }, + }, + }, + expected: expected{ + fileNameRegex: "^flag-variation-" + hostname + "-[0-9]*\\.parquet$", + featureEvents: []exporter.FeatureEvent{ + { + Kind: "feature", ContextKind: "anonymousUser", UserKey: "ABCD", CreationDate: 1617970547, Key: "random-key", + Variation: "Default", Value: `"YO"`, Default: false, + }, + { + Kind: "feature", ContextKind: "anonymousUser", UserKey: "EFGH", CreationDate: 1617970701, Key: "random-key", + Variation: "Default", Value: `"YO2"`, Default: false, Version: "127", + }, + }, + }, + }, { name: "custom CSV format", wantErr: false, @@ -104,6 +141,48 @@ func TestFile_Export(t *testing.T) { content: "./testdata/custom_csv_format.csv", }, }, + { + name: "complex parquet value", + wantErr: false, + fields: fields{ + Format: "parquet", + ParquetCompressionCodec: parquet.CompressionCodec_UNCOMPRESSED.String(), + }, + args: args{ + featureEvents: []exporter.FeatureEvent{ + { + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: map[string]interface{}{ + "string": "string", + "bool": true, + "float": 1.23, + "int": 1, + }, + Default: false, + }, + }, + }, + expected: expected{ + fileNameRegex: "^flag-variation-" + hostname + "-[0-9]*\\.parquet$", + featureEvents: []exporter.FeatureEvent{ + { + Kind: "feature", + ContextKind: "anonymousUser", + UserKey: "ABCD", + CreationDate: 1617970547, + Key: "random-key", + Variation: "Default", + Value: `{"bool":true,"float":1.23,"int":1,"string":"string"}`, + Default: false, + }, + }, + }, + }, { name: "custom file name", wantErr: false, @@ -209,6 +288,15 @@ func TestFile_Export(t *testing.T) { }, }, }, + { + name: "invalid parquet outputdir", + wantErr: true, + fields: fields{ + Format: "parquet", + OutputDir: "/tmp/foo/bar/", + }, + args: args{}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -219,10 +307,11 @@ func TestFile_Export(t *testing.T) { } f := &fileexporter.Exporter{ - Format: tt.fields.Format, - OutputDir: outputDir, - Filename: tt.fields.Filename, - CsvTemplate: tt.fields.CsvTemplate, + Format: tt.fields.Format, + OutputDir: outputDir, + Filename: tt.fields.Filename, + CsvTemplate: tt.fields.CsvTemplate, + ParquetCompressionCodec: tt.fields.ParquetCompressionCodec, } err := f.Export(context.Background(), tt.args.logger, tt.args.featureEvents) if tt.wantErr { @@ -234,6 +323,20 @@ func TestFile_Export(t *testing.T) { assert.Equal(t, 1, len(files), "Directory %s should have only one file", outputDir) assert.Regexp(t, tt.expected.fileNameRegex, files[0].Name(), "Invalid file name") + if tt.fields.Format == "parquet" { + fr, err := local.NewLocalFileReader(outputDir + "/" + files[0].Name()) + assert.NoError(t, err) + defer fr.Close() + pr, err := reader.NewParquetReader(fr, new(exporter.FeatureEvent), int64(runtime.NumCPU())) + assert.NoError(t, err) + defer pr.ReadStop() + gotFeatureEvents := make([]exporter.FeatureEvent, pr.GetNumRows()) + err = pr.Read(&gotFeatureEvents) + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expected.featureEvents, gotFeatureEvents) + return + } + expectedContent, _ := os.ReadFile(tt.expected.content) gotContent, _ := os.ReadFile(outputDir + "/" + files[0].Name()) assert.Equal(t, string(expectedContent), string(gotContent), "Wrong content in the output file") diff --git a/exporter/gcstorageexporter/exporter.go b/exporter/gcstorageexporter/exporter.go index 2e9f5db3f19..895bcea9bc5 100644 --- a/exporter/gcstorageexporter/exporter.go +++ b/exporter/gcstorageexporter/exporter.go @@ -24,7 +24,7 @@ type Exporter struct { Options []option.ClientOption // Format is the output format you want in your exported file. - // Available format are JSON and CSV. + // Available format are JSON, CSV, and Parquet. // Default: JSON Format string @@ -44,6 +44,11 @@ type Exporter struct { // Default: // {{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n CsvTemplate string + + // ParquetCompressionCodec is the parquet compression codec for better space efficiency. + // Available options https://github.com/apache/parquet-format/blob/master/Compression.md + // Default: SNAPPY + ParquetCompressionCodec string } func (f *Exporter) IsBulk() bool { @@ -72,10 +77,11 @@ func (f *Exporter) Export(ctx context.Context, logger *log.Logger, featureEvents // We call the File data exporter to get the file in the right format. // Files will be put in the temp directory, so we will be able to upload them to S3 from there. fileExporter := fileexporter.Exporter{ - Format: f.Format, - OutputDir: outputDir, - Filename: f.Filename, - CsvTemplate: f.CsvTemplate, + Format: f.Format, + OutputDir: outputDir, + Filename: f.Filename, + CsvTemplate: f.CsvTemplate, + ParquetCompressionCodec: f.ParquetCompressionCodec, } err = fileExporter.Export(ctx, logger, featureEvents) if err != nil { diff --git a/exporter/s3exporter/exporter.go b/exporter/s3exporter/exporter.go index a4bdd1b48e6..c7bc4fef347 100644 --- a/exporter/s3exporter/exporter.go +++ b/exporter/s3exporter/exporter.go @@ -26,7 +26,7 @@ type Exporter struct { AwsConfig *aws.Config // Format is the output format you want in your exported file. - // Available format are JSON and CSV. + // Available format are JSON, CSV and Parquet. // Default: JSON Format string @@ -47,6 +47,11 @@ type Exporter struct { // {{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n CsvTemplate string + // ParquetCompressionCodec is the parquet compression codec for better space efficiency. + // Available options https://github.com/apache/parquet-format/blob/master/Compression.md + // Default: SNAPPY + ParquetCompressionCodec string + s3Uploader s3manageriface.UploaderAPI init sync.Once } @@ -77,10 +82,11 @@ func (f *Exporter) Export(ctx context.Context, logger *log.Logger, featureEvents // We call the File data exporter to get the file in the right format. // Files will be put in the temp directory, so we will be able to upload them to Exporter from there. fileExporter := fileexporter.Exporter{ - Format: f.Format, - OutputDir: outputDir, - Filename: f.Filename, - CsvTemplate: f.CsvTemplate, + Format: f.Format, + OutputDir: outputDir, + Filename: f.Filename, + CsvTemplate: f.CsvTemplate, + ParquetCompressionCodec: f.ParquetCompressionCodec, } err = fileExporter.Export(ctx, logger, featureEvents) if err != nil { diff --git a/go.mod b/go.mod index 6eb83bc9523..d0326edc32d 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,8 @@ require ( github.com/stretchr/testify v1.8.2 github.com/swaggo/echo-swagger v1.4.0 github.com/swaggo/swag v1.8.12 + github.com/xitongsys/parquet-go v1.6.2 + github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 go.uber.org/zap v1.24.0 golang.org/x/net v0.9.0 golang.org/x/oauth2 v0.7.0 @@ -43,6 +45,8 @@ require ( cloud.google.com/go/pubsub v1.30.0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20220911224424-aa1f1f12a846 // indirect + github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 // indirect + github.com/apache/thrift v0.14.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -60,6 +64,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/gofuzz v1.1.0 // indirect github.com/google/renameio/v2 v2.0.0 // indirect @@ -72,6 +77,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.6 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -83,6 +89,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4/v4 v4.1.8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/xattr v0.4.9 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 6516c81b974..549cd6176ca 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,11 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20220911224424-aa1f1f12a846 h1:et5J11AOyUn9qwkIAF9kcxTxjTO8Z9oSmlOqH7MVSPo= github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20220911224424-aa1f1f12a846/go.mod h1:pSwJ0fSY5KhvocuWSx4fz3BA8OrA1bQn+K1Eli3BRwM= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516 h1:byKBBF2CKWBjjA4J1ZL2JXttJULvWSl50LegTyRZ728= +github.com/apache/arrow/go/arrow v0.0.0-20200730104253-651201b0f516/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0= +github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.14.2 h1:hY4rAyg7Eqbb27GB6gkhUKrRAuc8xRjlNtJq+LseKeY= +github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -82,6 +87,7 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/aws/aws-lambda-go v1.19.1/go.mod h1:jJmlefzPfGnckuHdXX7/80O3BvUUi12XOkbv4w9SGLU= github.com/aws/aws-lambda-go v1.40.0 h1:6dKcDpXsTpapfCFF6Debng6CiV/Z3sNHekM6bwhI2J0= github.com/aws/aws-lambda-go v1.40.0/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= +github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.44.244 h1:QzBWLD5HjZHdRZyTMTOWtD9Pobzf1n8/CeTJB4giXi0= github.com/aws/aws-sdk-go v1.44.244/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/awslabs/aws-lambda-go-api-proxy v0.14.0 h1:G+E4vjkw9roMIWsLKVmrDZxKEipJwoqkiiPUB2dtGqU= @@ -117,6 +123,7 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -196,6 +203,7 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= @@ -225,6 +233,7 @@ github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -244,10 +253,14 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -323,6 +336,7 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v0.0.0-20180228145832-27454136f036/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -343,8 +357,10 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/httpexpect/v2 v2.3.1/go.mod h1:ICTf89VBKSD3KB0fsyyHviKF8G8hyepP0dOXJPWz3T0= github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5UybogqrXvJTBE= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= +github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -370,10 +386,13 @@ github.com/kataras/tunnel v0.0.3/go.mod h1:VOlCoaUE5zN1buE+yAjWCkjfQ9hxGuhomKLsj github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -474,10 +493,13 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pborman/getopt v0.0.0-20180729010549-6fdd0a2c7117/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= +github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -526,6 +548,7 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= @@ -549,6 +572,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -598,6 +622,12 @@ github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xitongsys/parquet-go v1.5.1/go.mod h1:xUxwM8ELydxh4edHGegYq1pA8NnMKDx0K/GyB0o2bww= +github.com/xitongsys/parquet-go v1.6.2 h1:MhCaXii4eqceKPu9BwrjLqyK10oX9WF+xGhwvwbw7xM= +github.com/xitongsys/parquet-go v1.6.2/go.mod h1:IulAQyalCm0rPiZVNnCgm/PCL64X2tdSVGMQ/UeKqWA= +github.com/xitongsys/parquet-go-source v0.0.0-20190524061010-2b72cbee77d5/go.mod h1:xxCx7Wpym/3QCo6JhujJX51dzSXrwmb0oH6FQb39SEA= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 h1:a742S4V5A15F93smuVxA60LQWsrCnN8bKeWDBARU1/k= +github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0/go.mod h1:HYhIKsdns7xz80OgkbgJYrtQY7FjHWHKH6cvN7+czGE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= @@ -638,6 +668,7 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180723164146-c126467f60eb/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1082,6 +1113,11 @@ gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/website/docs/go_module/data_collection/file.md b/website/docs/go_module/data_collection/file.md index a606fb5a8ed..9bcf78aeb1a 100644 --- a/website/docs/go_module/data_collection/file.md +++ b/website/docs/go_module/data_collection/file.md @@ -30,8 +30,9 @@ ffclient.Config{ | Field | Description | |---|---| |`OutputDir` | OutputDir is the location of the directory where to store the exported files.
It should finish with a `/`. | -|`Format` | _(Optional)_ Format is the output format you want in your exported file.
Available format: **`JSON`**, **`CSV`**.
**Default: `JSON`** | +|`Format` | _(Optional)_ Format is the output format you want in your exported file.
Available format: **`JSON`**, **`CSV`**, **`Parquet`**.
**Default: `JSON`** | |`Filename` | _(Optional)_ Filename is the name of your output file.
You can use a templated config to define the name of your exported files.
Available replacement are `{{ .Hostname}}`, `{{ .Timestamp}}` and `{{ .Format}}`
**Default: `flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}`**| |`CsvTemplate` | _(Optional)_ CsvTemplate is used if your output format is CSV.
This field will be ignored if you are using another format than CSV.
You can decide which fields you want in your CSV line with a go-template syntax, please check [internal/exporter/feature_event.go](https://github.com/thomaspoignant/go-feature-flag/blob/main/internal/exporter/feature_event.go) to see what are the fields available.
**Default:** `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n` | +| `ParquetCompressionCodec` | _(Optional)_ ParquetCompressionCodec is the parquet compression codec for better space efficiency. [Available options](https://github.com/apache/parquet-format/blob/master/Compression.md)
**Default: `SNAPPY`** |` Check the [godoc for full details](https://pkg.go.dev/github.com/thomaspoignant/go-feature-flag/exporter/fileexporter). diff --git a/website/docs/go_module/data_collection/google_cloud_storage.md b/website/docs/go_module/data_collection/google_cloud_storage.md index fa394e3f1df..0e1821453c8 100644 --- a/website/docs/go_module/data_collection/google_cloud_storage.md +++ b/website/docs/go_module/data_collection/google_cloud_storage.md @@ -38,8 +38,9 @@ ffclient.Config{ | `Bucket ` | Name of your Google Cloud Storage Bucket. | | `CsvTemplate` | *(optional)* CsvTemplate is used if your output format is CSV. This field will be ignored if you are using another format than CSV. You can decide which fields you want in your CSV line with a go-template syntax, please check [internal/exporter/feature_event.go](https://github.com/thomaspoignant/go-feature-flag/blob/main/internal/exporter/feature_event.go) to see what are the fields available.
**Default:** `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n` | | `Filename` | *(optional)* Filename is the name of your output file. You can use a templated config to define the name of your exported files.
Available replacement are `{{ .Hostname}}`, `{{ .Timestamp}`} and `{{ .Format}}`
Default: `flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}` | -| `Format` | *(optional)* Format is the output format you want in your exported file. Available format are **`JSON`** and **`CSV`**. *(Default: `JSON`)* | +| `Format` | *(optional)* Format is the output format you want in your exported file. Available format are **`JSON`**, **`CSV`**, **`Parquet`**. *(Default: `JSON`)* | | `Options` | *(optional)* An instance of `option.ClientOption` that configures your access to Google Cloud.
Check [this documentation for more info](https://cloud.google.com/docs/authentication). | | `Path ` | *(optional)* The location of the directory in your bucket. | +| `ParquetCompressionCodec` | *(optional)* ParquetCompressionCodec is the parquet compression codec for better space efficiency. [Available options](https://github.com/apache/parquet-format/blob/master/Compression.md) *(Default: `SNAPPY`)* |` Check the [godoc for full details](https://pkg.go.dev/github.com/thomaspoignant/go-feature-flag/exporter/gcstorageexporter). diff --git a/website/docs/go_module/data_collection/s3.md b/website/docs/go_module/data_collection/s3.md index 16208070dd3..4a9fa3aba01 100644 --- a/website/docs/go_module/data_collection/s3.md +++ b/website/docs/go_module/data_collection/s3.md @@ -46,7 +46,8 @@ ffclient.Config{ | `AwsConfig ` | An instance of `aws.Config` that configure your access to AWS *(see [this documentation for more info](https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/configuring-sdk.html))*. | | `CsvTemplate` | *(optional)* CsvTemplate is used if your output format is CSV. This field will be ignored if you are using another format than CSV. You can decide which fields you want in your CSV line with a go-template syntax, please check [internal/exporter/feature_event.go](https://github.com/thomaspoignant/go-feature-flag/blob/main/internal/exporter/feature_event.go) to see what are the fields available.
**Default:** `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n` | | `Filename` | *(optional)* Filename is the name of your output file. You can use a templated config to define the name of your exported files.
Available replacement are `{{ .Hostname}}`, `{{ .Timestamp}`} and `{{ .Format}}`
Default: `flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}` | -| `Format` | *(optional)* Format is the output format you want in your exported file. Available format are **`JSON`** and **`CSV`**. *(Default: `JSON`)* | +| `Format` | *(optional)* Format is the output format you want in your exported file. Available format are **`JSON`**, **`CSV`**, **`Parquet`**. *(Default: `JSON`)* | | `S3Path ` | *(optional)* The location of the directory in S3. | +| `ParquetCompressionCodec` | *(optional)* ParquetCompressionCodec is the parquet compression codec for better space efficiency. [Available options](https://github.com/apache/parquet-format/blob/master/Compression.md) *(Default: `SNAPPY`)* |` Check the [godoc for full details](https://pkg.go.dev/github.com/thomaspoignant/go-feature-flag/exporter/s3exporter). diff --git a/website/docs/relay_proxy/configure_relay_proxy.md b/website/docs/relay_proxy/configure_relay_proxy.md index e80a888a2d9..a69813e19c0 100644 --- a/website/docs/relay_proxy/configure_relay_proxy.md +++ b/website/docs/relay_proxy/configure_relay_proxy.md @@ -31,7 +31,7 @@ If you want to replace a nested fields, please use `_` to separate each field _( | `startWithRetrieverError` | boolean | `false` | By default the **relay proxy** will crash if he is not able to retrieve the flags from the configuration.
If you don't want your relay proxy to crash, you can set `startWithRetrieverError` to true. Until the flag is retrievable the relay proxy will only answer with default values. | | `exporter` | [exporter](#exporter) | **none** | Exporter is the configuration on how to export data. | | `notifier` | [notifier](#notifier) | **none** | Notifiers is the configuration on where to notify a flag change. | -| `apiKeys` | []string | **none** | List of authorized API keys. Each request will need to provide one of authorized key inside `Authorization` header with format `Bearer `. There will be no authorization when this config is not configured.

_Note: there will be no authorization when this config is not set._ | +| `apiKeys` | []string | **none** | List of authorized API keys. Each request will need to provide one of authorized key inside `Authorization` header with format `Bearer `.

_Note: there will be no authorization when this config is not set._ | @@ -125,9 +125,10 @@ _Note that relay proxy is only supporting this while running inside the kubernet | `outputDir` | string | **none** | **(mandatory)** OutputDir is the location of the directory where to store the exported files. It should finish with a `/`. | | `flushInterval` | int | `60000` | The interval in millisecond between 2 calls to the webhook _(if the `maxEventInMemory` is reached before the flushInterval we will call the webhook before)_. | | `maxEventInMemory` | int | `100000` | If we hit that limit we will call the webhook. | -| `format` | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`. | +| `format` | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`, `Parquet`. | | `filename` | string | `flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}` | You can use a templated config to define the name of your exported files. Available replacement are `{{ .Hostname}}`, `{{ .Timestamp}}` and `{{ .Format}` | | `csvTemplate` | string | `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n` | CsvTemplate is used if your output format is CSV.
This field will be ignored if you are using another format than CSV.
You can decide which fields you want in your CSV line with a go-template syntax, please check [`internal/exporter/feature_event.go`](https://github.com/thomaspoignant/go-feature-flag/blob/main/internal/exporter/feature_event.go) to see what are the fields available. |` +| `parquetCompressionCodec` | string | `SNAPPY` | ParquetCompressionCodec is the parquet compression codec for better space efficiency. [Available options](https://github.com/apache/parquet-format/blob/master/Compression.md) |` ### Log @@ -147,10 +148,11 @@ _Note that relay proxy is only supporting this while running inside the kubernet | `bucket` | string | **none** | **(mandatory)** Name of your S3 Bucket. | | `flushInterval` | int | `60000` | The interval in millisecond between 2 calls to the webhook _(if the `maxEventInMemory` is reached before the flushInterval we will call the webhook before)_. | | `maxEventInMemory` | int | `100000` | If we hit that limit we will call the webhook. | -| `format` | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`. | +| `format` | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`, `Parquet`. | | `filename` | string | `flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}` | You can use a templated config to define the name of your exported files. Available replacement are `{{ .Hostname}}`, `{{ .Timestamp}}` and `{{ .Format}` | | `csvTemplate` | string | `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n` | CsvTemplate is used if your output format is CSV.
This field will be ignored if you are using another format than CSV.
You can decide which fields you want in your CSV line with a go-template syntax, please check [`internal/exporter/feature_event.go`](https://github.com/thomaspoignant/go-feature-flag/blob/main/internal/exporter/feature_event.go) to see what are the fields available. |` | `path` | string | **bucket root level** | The location of the directory in S3. | +| `parquetCompressionCodec` | string | `SNAPPY` | ParquetCompressionCodec is the parquet compression codec for better space efficiency. [Available options](https://github.com/apache/parquet-format/blob/master/Compression.md) |` ### Google Storage @@ -160,10 +162,11 @@ _Note that relay proxy is only supporting this while running inside the kubernet | `bucket` | string | **none** | **(mandatory)** Name of your Google Cloud Storage Bucket. | | `flushInterval` | int | `60000` | The interval in millisecond between 2 calls to the webhook _(if the `maxEventInMemory` is reached before the flushInterval we will call the webhook before)_. | | `maxEventInMemory` | int | `100000` | If we hit that limit we will call the webhook. | -| `format` | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`. | +| `format` | string | `JSON` | Format is the output format you want in your exported file. Available format: `JSON`, `CSV`, `Parquet`. | | `filename` | string | `flag-variation-{{ .Hostname}}-{{ .Timestamp}}.{{ .Format}}` | You can use a templated config to define the name of your exported files. Available replacement are `{{ .Hostname}}`, `{{ .Timestamp}}` and `{{ .Format}` | | `csvTemplate` | string | `{{ .Kind}};{{ .ContextKind}};{{ .UserKey}};{{ .CreationDate}};{{ .Key}};{{ .Variation}};{{ .Value}};{{ .Default}}\n` | CsvTemplate is used if your output format is CSV.
This field will be ignored if you are using another format than CSV.
You can decide which fields you want in your CSV line with a go-template syntax, please check [`internal/exporter/feature_event.go`](https://github.com/thomaspoignant/go-feature-flag/blob/main/internal/exporter/feature_event.go) to see what are the fields available. |` | `path` | string | **bucket root level** | The location of the directory in S3. | +| `parquetCompressionCodec` | string | `SNAPPY` | ParquetCompressionCodec is the parquet compression codec for better space efficiency. [Available options](https://github.com/apache/parquet-format/blob/master/Compression.md) |`