From 115da668c76b173e95687bcc3ed2a2a45a1e40ed Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 9 Oct 2022 10:03:01 -0400 Subject: [PATCH 001/100] Taking a stab at a way to standardize viper usage Signed-off-by: Andrew Mason --- go/viperutil/viper.go | 186 +++++++++++++++++++ go/vt/mysqlctl/azblobbackupstorage/azblob.go | 85 ++++++--- 2 files changed, 248 insertions(+), 23 deletions(-) create mode 100644 go/viperutil/viper.go diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go new file mode 100644 index 00000000000..5fb4a13250e --- /dev/null +++ b/go/viperutil/viper.go @@ -0,0 +1,186 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package viperutil provides additional functions for Vitess's use of viper for +// managing configuration of various components. +package viperutil + +import ( + "fmt" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +type Option[T any] func(val *Value[T]) + +func WithAliases[T any](aliases ...string) Option[T] { + return func(val *Value[T]) { + val.aliases = append(val.aliases, aliases...) + } +} + +func WithDefault[T any](t T) Option[T] { + return func(val *Value[T]) { + val.hasDefault, val.value = true, t + } +} + +func WithEnvVars[T any](envvars ...string) Option[T] { + return func(val *Value[T]) { + val.envvars = append(val.envvars, envvars...) + } +} + +func WithFlags[T any](flags ...string) Option[T] { + return func(val *Value[T]) { + val.flagNames = append(val.flagNames, flags...) + } +} + +type Value[T any] struct { + key string + value T + + aliases []string + flagNames []string + envvars []string + hasDefault bool + + resolve func(string) T + loaded bool +} + +// NewValue returns a viper-backed value with the given key, lookup function, +// and bind options. +func NewValue[T any](key string, getFunc func(string) T, opts ...Option[T]) *Value[T] { + val := &Value[T]{ + key: key, + resolve: getFunc, + } + + for _, opt := range opts { + opt(val) + } + + return val +} + +// Bind binds the value to flags, aliases, and envvars, depending on the Options +// the value was created with. +// +// If the passed-in viper is nil, the global viper instance is used. +// +// If the passed-in flag set is nil, flag binding is skipped. Otherwise, if any +// flag names do not match the value's canonical key, they are registered as +// additional aliases for the value's key. This function panics if a flag name +// is specified that is not defined on the flag set. +func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { + if v == nil { + v = viper.GetViper() + } + + aliases := val.aliases + + // Bind any flags, additionally aliasing flag names to the canonical key for + // this value. + if fs != nil { + for _, name := range val.flagNames { + f := fs.Lookup(name) + if f == nil { + // TODO: Don't love "configuration error" here as it might make + // users think _they_ made a config error, but really it is us, + // the Vitess authors, that have misconfigured a flag binding in + // the source code somewhere. + panic(fmt.Sprintf("configuration error: attempted to bind %s to unspecified flag %s", val.key, name)) + } + + if f.Name != val.key { + aliases = append(aliases, f.Name) + } + + // We are deliberately ignoring the error value here because it + // only occurs when `f == nil` and we check for that ourselves. + _ = v.BindPFlag(val.key, f) + } + } + + // Bind any aliases. + for _, alias := range aliases { + v.RegisterAlias(alias, val.key) + } + + // Bind any envvars. + if len(val.envvars) > 0 { + vars := append([]string{val.key}, val.envvars...) + // Again, ignoring the error value here, which is non-nil only when + // `len(vars) == 0`. + _ = v.BindEnv(vars...) + } + + // Set default value. + if val.hasDefault { + v.SetDefault(val.key, val.value) + } +} + +// Fetch returns the underlying value from the backing viper. +func (val *Value[T]) Fetch() T { + val.value, val.loaded = val.resolve(val.key), true + return val.value +} + +// Get returns the underlying value, loading from the backing viper only on +// first use. For dynamic reloading, use Fetch. +func (val *Value[T]) Get() T { + if !val.loaded { + val.Fetch() + } + + return val.value +} + +// Value returns the current underlying value. If a value has not been +// previously loaded (either via Fetch or Get), and the value was not created +// with a WithDefault option, the return value is whatever the zero value for +// the type T is. +func (val *Value[T]) Value() T { + return val.value +} + +/* +old api + +func (v *Value[T]) BindFlag(f *pflag.Flag, aliases ...string) { + BindFlagWithAliases(f, v.key, aliases...) +} + +func (v *Value[T]) BindEnv(envvars ...string) { + viper.BindEnv(append([]string{v.key}, envvars...)...) +} + +func BindFlagWithAliases(f *pflag.Flag, canonicalKey string, aliases ...string) { + viper.BindPFlag(canonicalKey, f) + if canonicalKey != f.Name { + aliases = append(aliases, f.Name) + } + + for _, alias := range aliases { + viper.RegisterAlias(alias, canonicalKey) + } +} + +*/ diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 08fa24643b7..e9968c4ce08 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -31,7 +31,9 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/spf13/pflag" + "github.com/spf13/viper" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" @@ -40,26 +42,67 @@ import ( var ( // This is the account name - accountName string + accountName = viperutil.NewValue( + configKey("account.name"), + viper.GetString, + viperutil.WithFlags[string]("azblob_backup_account_name"), + viperutil.WithEnvVars[string]("VT_AZBLOB_ACCOUNT_NAME"), + ) // This is the private access key - accountKeyFile string + accountKeyFile = viperutil.NewValue( + configKey("account.key_file"), + viper.GetString, + viperutil.WithFlags[string]("azblob_backup_account_key_file"), + ) // This is the name of the container that will store the backups - containerName string + containerName = viperutil.NewValue( + configKey("container_name"), + viper.GetString, + viperutil.WithFlags[string]("azblob_backup_container_name"), + ) // This is an optional prefix to prepend to all files - storageRoot string - - azBlobParallelism int + storageRoot = viperutil.NewValue( + configKey("storage_root"), + viper.GetString, + viperutil.WithFlags[string]("azblob_backup_storage_root"), + ) + + azBlobParallelism = viperutil.NewValue( + configKey("parallelism"), + viper.GetInt, + viperutil.WithFlags[int]("azblob_backup_parallelism"), + ) ) +const configKeyPrefix = "backup.storage.azblob" + +func configKey(key string) string { + parts := []string{configKeyPrefix} + if key != "" { + parts = append(parts, key) + } + + return strings.Join(parts, ".") +} + func registerFlags(fs *pflag.FlagSet) { - fs.StringVar(&accountName, "azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") - fs.StringVar(&accountKeyFile, "azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") - fs.StringVar(&containerName, "azblob_backup_container_name", "", "Azure Blob Container Name.") - fs.StringVar(&storageRoot, "azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") - fs.IntVar(&azBlobParallelism, "azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased).") + fs.String("azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") + accountName.Bind(nil, fs) + + fs.String("azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") + accountKeyFile.Bind(nil, fs) + + fs.String("azblob_backup_container_name", "", "Azure Blob Container Name.") + containerName.Bind(nil, fs) + + fs.String("azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") + storageRoot.Bind(nil, fs) + + fs.Int("azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased).") + azBlobParallelism.Bind(nil, fs) } func init() { @@ -79,16 +122,12 @@ const ( // 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key) // 2. Environment variables func azInternalCredentials() (string, string, error) { - actName := accountName - if actName == "" { - // Check the Environmental Value - actName = os.Getenv("VT_AZBLOB_ACCOUNT_NAME") - } + actName := accountName.Get() var actKey string - if accountKeyFile != "" { - log.Infof("Getting Azure Storage Account key from file: %s", accountKeyFile) - dat, err := os.ReadFile(accountKeyFile) + if keyFile := accountKeyFile.Get(); keyFile != "" { + log.Infof("Getting Azure Storage Account key from file: %s", keyFile) + dat, err := os.ReadFile(keyFile) if err != nil { return "", "", err } @@ -219,7 +258,7 @@ func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, file defer bh.waitGroup.Done() _, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{ BufferSize: azblob.BlockBlobMaxStageBlockBytes, - MaxBuffers: azBlobParallelism, + MaxBuffers: azBlobParallelism.Get(), }) if err != nil { reader.CloseWithError(err) @@ -286,7 +325,7 @@ func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) { if err != nil { return nil, err } - u := azServiceURL(credentials).NewContainerURL(containerName) + u := azServiceURL(credentials).NewContainerURL(containerName.Get()) return &u, nil } @@ -425,8 +464,8 @@ func (bs *AZBlobBackupStorage) WithParams(params backupstorage.Params) backupsto // Unlike path.Join, it doesn't collapse ".." or strip trailing slashes. // It also adds the value of the -azblob_backup_storage_root flag if set. func objName(parts ...string) string { - if storageRoot != "" { - return storageRoot + "/" + strings.Join(parts, "/") + if root := storageRoot.Get(); root != "" { + return root + "/" + strings.Join(parts, "/") } return strings.Join(parts, "/") } From 5b40343ba4ac8dc86e5726388cb622013bab0b71 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 10 Oct 2022 09:21:38 -0400 Subject: [PATCH 002/100] port go/trace to viper-bound flags Signed-off-by: Andrew Mason --- go/trace/plugin_datadog.go | 31 +++++++--- go/trace/plugin_jaeger.go | 64 +++++++++++--------- go/trace/trace.go | 35 ++++++++--- go/trace/trace_test.go | 14 ++++- go/viperutil/key.go | 50 +++++++++++++++ go/vt/mysqlctl/azblobbackupstorage/azblob.go | 9 +-- 6 files changed, 150 insertions(+), 53 deletions(-) create mode 100644 go/viperutil/key.go diff --git a/go/trace/plugin_datadog.go b/go/trace/plugin_datadog.go index 25f0b6894cd..53ab85ae8b2 100644 --- a/go/trace/plugin_datadog.go +++ b/go/trace/plugin_datadog.go @@ -6,37 +6,54 @@ import ( "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" + "github.com/spf13/viper" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer" ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + + "vitess.io/vitess/go/viperutil" ) var ( - dataDogHost string - dataDogPort string + dataDogConfigKey = viperutil.KeyPartial(configKey("datadog")) + + dataDogHost = viperutil.NewValue( + dataDogConfigKey("agent.host"), + viper.GetString, + viperutil.WithFlags[string]("datadog-agent-host"), + ) + dataDogPort = viperutil.NewValue( + dataDogConfigKey("agent.port"), + viper.GetString, + viperutil.WithFlags[string]("datadog-agent-port"), + ) ) func init() { // If compiled with plugin_datadaog, ensure that trace.RegisterFlags // includes datadaog tracing flags. pluginFlags = append(pluginFlags, func(fs *pflag.FlagSet) { - fs.StringVar(&dataDogHost, "datadog-agent-host", "", "host to send spans to. if empty, no tracing will be done") - fs.StringVar(&dataDogPort, "datadog-agent-port", "", "port to send spans to. if empty, no tracing will be done") + fs.String("datadog-agent-host", "", "host to send spans to. if empty, no tracing will be done") + dataDogHost.Bind(nil, fs) + + fs.String("datadog-agent-port", "", "port to send spans to. if empty, no tracing will be done") + dataDogPort.Bind(nil, fs) }) } func newDatadogTracer(serviceName string) (tracingService, io.Closer, error) { - if dataDogHost == "" || dataDogPort == "" { + host, port := dataDogHost.Get(), dataDogPort.Get() + if host == "" || port == "" { return nil, nil, fmt.Errorf("need host and port to datadog agent to use datadog tracing") } opts := []ddtracer.StartOption{ - ddtracer.WithAgentAddr(dataDogHost + ":" + dataDogPort), + ddtracer.WithAgentAddr(host + ":" + port), ddtracer.WithServiceName(serviceName), ddtracer.WithDebugMode(true), ddtracer.WithSampler(ddtracer.NewRateSampler(samplingRate.Get())), } - if enableLogging { + if enableLogging.Get() { opts = append(opts, ddtracer.WithLogger(&traceLogger{})) } diff --git a/go/trace/plugin_jaeger.go b/go/trace/plugin_jaeger.go index b1449bc576f..a4edd247378 100644 --- a/go/trace/plugin_jaeger.go +++ b/go/trace/plugin_jaeger.go @@ -18,14 +18,13 @@ package trace import ( "io" - "os" "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" - "github.com/uber/jaeger-client-go" + "github.com/spf13/viper" "github.com/uber/jaeger-client-go/config" - "vitess.io/vitess/go/flagutil" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" ) @@ -36,18 +35,40 @@ included but nothing Jaeger specific. */ var ( - agentHost string - samplingType = flagutil.NewOptionalString("const") - samplingRate = flagutil.NewOptionalFloat64(0.1) + jaegerConfigKey = viperutil.KeyPartial(configKey("jaeger")) + agentHost = viperutil.NewValue( + jaegerConfigKey("agent-host"), + viper.GetString, + viperutil.WithFlags[string]("jaeger-agent-host"), + ) + samplingType = viperutil.NewValue( + jaegerConfigKey("sampling_type"), + viper.GetString, + viperutil.WithFlags[string]("tracing-sampling-type"), + viperutil.WithDefault("const"), + viperutil.WithEnvVars[string]("JAEGER_SAMPLER_TYPE"), + ) + samplingRate = viperutil.NewValue( + jaegerConfigKey("sampling_rate"), + viper.GetFloat64, + viperutil.WithFlags[float64]("tracing-sampling-rate"), + viperutil.WithDefault(0.1), + viperutil.WithEnvVars[float64]("JAEGER_SAMPLER_PARAM"), + ) ) func init() { // If compiled with plugin_jaeger, ensure that trace.RegisterFlags includes // jaeger tracing flags. pluginFlags = append(pluginFlags, func(fs *pflag.FlagSet) { - fs.StringVar(&agentHost, "jaeger-agent-host", "", "host and port to send spans to. if empty, no tracing will be done") - fs.Var(samplingType, "tracing-sampling-type", "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") - fs.Var(samplingRate, "tracing-sampling-rate", "sampling rate for the probabilistic jaeger sampler") + fs.String("jaeger-agent-host", "", "host and port to send spans to. if empty, no tracing will be done") + agentHost.Bind(nil, fs) + + fs.String("tracing-sampling-type", samplingType.Value(), "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") + samplingType.Bind(nil, fs) + + fs.Float64("tracing-sampling-rate", samplingRate.Value(), "sampling rate for the probabilistic jaeger sampler") + samplingRate.Bind(nil, fs) }) } @@ -79,32 +100,17 @@ func newJagerTracerFromEnv(serviceName string) (tracingService, io.Closer, error } // Allow command line args to override environment variables. - if agentHost != "" { - cfg.Reporter.LocalAgentHostPort = agentHost + if host := agentHost.Get(); host != "" { + cfg.Reporter.LocalAgentHostPort = host } log.Infof("Tracing to: %v as %v", cfg.Reporter.LocalAgentHostPort, cfg.ServiceName) - if os.Getenv("JAEGER_SAMPLER_PARAM") == "" { - // If the environment variable was not set, we take the flag regardless - // of whether it was explicitly set on the command line. - cfg.Sampler.Param = samplingRate.Get() - } else if samplingRate.IsSet() { - // If the environment variable was set, but the user also explicitly - // passed the command line flag, the flag takes precedence. - cfg.Sampler.Param = samplingRate.Get() - } - - if samplingType.IsSet() { - cfg.Sampler.Type = samplingType.Get() - } else if cfg.Sampler.Type == "" { - log.Infof("--tracing-sampler-type was not set, and JAEGER_SAMPLER_TYPE was not set, defaulting to const sampler") - cfg.Sampler.Type = jaeger.SamplerTypeConst - } - + cfg.Sampler.Param = samplingRate.Get() + cfg.Sampler.Type = samplingType.Get() log.Infof("Tracing sampler type %v (param: %v)", cfg.Sampler.Type, cfg.Sampler.Param) var opts []config.Option - if enableLogging { + if enableLogging.Get() { opts = append(opts, config.Logger(&traceLogger{})) } else if cfg.Reporter.LogSpans { log.Warningf("JAEGER_REPORTER_LOG_SPANS was set, but --tracing-enable-logging was not; spans will not be logged") diff --git a/go/trace/trace.go b/go/trace/trace.go index dd0e1b56b62..6892da815fc 100644 --- a/go/trace/trace.go +++ b/go/trace/trace.go @@ -26,8 +26,10 @@ import ( "strings" "github.com/spf13/pflag" + "github.com/spf13/viper" "google.golang.org/grpc" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" ) @@ -125,6 +127,8 @@ type tracingService interface { // object to make sure that all spans are sent to the backend before the process exits. type TracerFactory func(serviceName string) (tracingService, io.Closer, error) +const configKeyPrefix = "trace" + var ( // tracingBackendFactories should be added to by a plugin during init() to install itself tracingBackendFactories = make(map[string]TracerFactory) @@ -133,15 +137,29 @@ var ( /* flags */ - tracingServer = "noop" - enableLogging bool + configKey = viperutil.KeyPartial(configKeyPrefix) + + tracingServer = viperutil.NewValue( + configKey("service"), + viper.GetString, + viperutil.WithFlags[string]("tracer"), + viperutil.WithDefault("noop"), + ) + enableLogging = viperutil.NewValue( + configKey("enable-logging"), + viper.GetBool, + viperutil.WithFlags[bool]("tracing-enable-logging"), + ) pluginFlags []func(fs *pflag.FlagSet) ) func RegisterFlags(fs *pflag.FlagSet) { - fs.StringVar(&tracingServer, "tracer", "noop", "tracing service to use") - fs.BoolVar(&enableLogging, "tracing-enable-logging", false, "whether to enable logging in the tracing service") + fs.String("tracer", tracingServer.Value(), "tracing service to use") + tracingServer.Bind(nil, fs) + + fs.Bool("tracing-enable-logging", false, "whether to enable logging in the tracing service") + enableLogging.Bind(nil, fs) for _, fn := range pluginFlags { fn(fs) @@ -150,20 +168,21 @@ func RegisterFlags(fs *pflag.FlagSet) { // StartTracing enables tracing for a named service func StartTracing(serviceName string) io.Closer { - factory, ok := tracingBackendFactories[tracingServer] + tracingBackend := tracingServer.Get() + factory, ok := tracingBackendFactories[tracingBackend] if !ok { return fail(serviceName) } tracer, closer, err := factory(serviceName) if err != nil { - log.Error(vterrors.Wrapf(err, "failed to create a %s tracer", tracingServer)) + log.Error(vterrors.Wrapf(err, "failed to create a %s tracer", tracingBackend)) return &nilCloser{} } currentTracer = tracer - if tracingServer != "noop" { - log.Infof("successfully started tracing with [%s]", tracingServer) + if tracingBackend != "noop" { + log.Infof("successfully started tracing with [%s]", tracingBackend) } return closer diff --git a/go/trace/trace_test.go b/go/trace/trace_test.go index 35500c93b4e..7649c4f65c0 100644 --- a/go/trace/trace_test.go +++ b/go/trace/trace_test.go @@ -23,7 +23,10 @@ import ( "strings" "testing" + "github.com/spf13/viper" "google.golang.org/grpc" + + "vitess.io/vitess/go/viperutil" ) func TestFakeSpan(t *testing.T) { @@ -49,7 +52,16 @@ func TestRegisterService(t *testing.T) { return tracer, tracer, nil } - tracingServer = fakeName + var old viperutil.Value[string] + old = *tracingServer + t.Cleanup(func() { + *tracingServer = old + }) + + v := viper.New() + tracingServer = viperutil.NewValue(configKey("service"), v.GetString) + v.Set(configKey("service"), fakeName) + tracingServer.Fetch() serviceName := "vtservice" closer := StartTracing(serviceName) diff --git a/go/viperutil/key.go b/go/viperutil/key.go new file mode 100644 index 00000000000..4de76210710 --- /dev/null +++ b/go/viperutil/key.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import "strings" + +// Key returns a full key given a prefix and optional key. If key is the empty +// string, only the prefix is returned, otherwise the prefix is joined with the +// key with ".", the default viper key delimiter. +func Key(prefix string, key string) string { + parts := []string{prefix} + if key != "" { + parts = append(parts, key) + } + + return strings.Join(parts, ".") +} + +// KeyPartial returns a partial func that returns full keys joined to the given +// prefix. +// +// Packages can use this to avoid repeating a prefix across all their key +// accesses, for example: +// +// package mypkg +// var ( +// configKey = viperutil.KeyPartial("foo.bar.mypkg") +// v1 = viperutil.NewValue(configKey("v1"), ...) // bound to "foo.bar.mypkg.v1" +// v2 = viperutil.NewValue(configKey("v2"), ...) // bound to "foo.bar.mypkg.v2" +// ... +// ) +func KeyPartial(prefix string) func(string) string { + return func(key string) string { + return Key(prefix, key) + } +} diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index e9968c4ce08..10134c5b73d 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -79,14 +79,7 @@ var ( const configKeyPrefix = "backup.storage.azblob" -func configKey(key string) string { - parts := []string{configKeyPrefix} - if key != "" { - parts = append(parts, key) - } - - return strings.Join(parts, ".") -} +var configKey = viperutil.KeyPartial(configKeyPrefix) func registerFlags(fs *pflag.FlagSet) { fs.String("azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") From 41dd81cb8a4c043766d91cfba0475090cb78de70 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 12 Oct 2022 14:43:50 -0400 Subject: [PATCH 003/100] add viperget subpackage Signed-off-by: Andrew Mason --- go/viperutil/viperget/doc.go | 30 ++++++++++++++ go/viperutil/viperget/duration_or_int.go | 41 ++++++++++++++++++ go/viperutil/viperget/tablet_type.go | 53 ++++++++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 go/viperutil/viperget/doc.go create mode 100644 go/viperutil/viperget/duration_or_int.go create mode 100644 go/viperutil/viperget/tablet_type.go diff --git a/go/viperutil/viperget/doc.go b/go/viperutil/viperget/doc.go new file mode 100644 index 00000000000..147285d62d7 --- /dev/null +++ b/go/viperutil/viperget/doc.go @@ -0,0 +1,30 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +package viperget provides custom getter functions for retrieving values from a +viper. + +For example, to retrieve a key as a topodatapb.TabletType variable, the +viperutil.Value should be instantiated like: + + // Passing `nil` to viperget.TabletType will have it lookup the key on the + // current global viper. + viperutil.NewValue[topodatapb.TabletType](myKey, viperget.TabletType(nil), opts...) + // If using a specific viper instance, instead do the following. + viperutil.NewValue[topodatapb.TabletType](myKey, viperget.TabletType(myViper), opts...) +*/ +package viperget diff --git a/go/viperutil/viperget/duration_or_int.go b/go/viperutil/viperget/duration_or_int.go new file mode 100644 index 00000000000..463048df0a7 --- /dev/null +++ b/go/viperutil/viperget/duration_or_int.go @@ -0,0 +1,41 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperget + +import ( + "time" + + "github.com/spf13/viper" +) + +func DurationOrInt(v *viper.Viper, intFallbackUnit time.Duration) func(key string) time.Duration { + if v == nil { + v = viper.GetViper() + } + + return func(key string) time.Duration { + if !v.IsSet(key) { + return time.Duration(0) + } + + if d := v.GetDuration(key); d != 0 { + return d + } + + return time.Duration(v.GetInt(key)) * intFallbackUnit + } +} diff --git a/go/viperutil/viperget/tablet_type.go b/go/viperutil/viperget/tablet_type.go new file mode 100644 index 00000000000..71088d3f357 --- /dev/null +++ b/go/viperutil/viperget/tablet_type.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperget + +import ( + "github.com/spf13/viper" + + "vitess.io/vitess/go/vt/log" + topodatapb "vitess.io/vitess/go/vt/proto/topodata" + "vitess.io/vitess/go/vt/topo/topoproto" +) + +func TabletType(v *viper.Viper) func(key string) topodatapb.TabletType { + if v == nil { + v = viper.GetViper() + } + + return func(key string) (ttype topodatapb.TabletType) { + if !v.IsSet(key) { + return + } + + switch val := v.Get(key).(type) { + case int: + ttype = topodatapb.TabletType(int32(val)) + case string: + var err error + if ttype, err = topoproto.ParseTabletType(val); err != nil { + // TODO: decide how we want to handle these cases. + log.Warningf("GetTabletTypeValue failed to parse tablet type: %s. Defaulting to TabletType %s.\n", err.Error(), topoproto.TabletTypeLString(ttype)) + } + default: + // TODO: decide how we want to handle this case. + log.Warningf("GetTabletTypeValue: invalid Go type %T for TabletType; must be string or int. Defaulting to TabletType %s.\n", val, ttype) + } + + return + } +} From 893e0009831126826ab558cdb6be2ec7910f56f5 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 12 Oct 2022 14:44:34 -0400 Subject: [PATCH 004/100] add viperget subpackage Signed-off-by: Andrew Mason --- go/viperutil/viperget/doc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/viperutil/viperget/doc.go b/go/viperutil/viperget/doc.go index 147285d62d7..a99c45195ac 100644 --- a/go/viperutil/viperget/doc.go +++ b/go/viperutil/viperget/doc.go @@ -26,5 +26,8 @@ viperutil.Value should be instantiated like: viperutil.NewValue[topodatapb.TabletType](myKey, viperget.TabletType(nil), opts...) // If using a specific viper instance, instead do the following. viperutil.NewValue[topodatapb.TabletType](myKey, viperget.TabletType(myViper), opts...) + +This is a subpackage instead of being part of viperutil directly in order to +avoid an import cycle between go/vt/log and viperutil. */ package viperget From 6a65f862c8b7bfc560f3a3f1633f7a4c6b728cad Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 13 Oct 2022 06:18:35 -0400 Subject: [PATCH 005/100] Ensure viper values are bound prior to access Signed-off-by: Andrew Mason --- go/viperutil/viper.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index 5fb4a13250e..f50dee2ef2b 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -61,6 +61,7 @@ type Value[T any] struct { hasDefault bool resolve func(string) T + bound bool loaded bool } @@ -89,6 +90,12 @@ func NewValue[T any](key string, getFunc func(string) T, opts ...Option[T]) *Val // additional aliases for the value's key. This function panics if a flag name // is specified that is not defined on the flag set. func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { + if val.bound { + return + } + + val.bound = true + if v == nil { v = viper.GetViper() } @@ -139,6 +146,9 @@ func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { // Fetch returns the underlying value from the backing viper. func (val *Value[T]) Fetch() T { + // Prior to fetching anything, bind the value to the global viper if it + // hasn't been bound to some viper already. + val.Bind(nil, nil) val.value, val.loaded = val.resolve(val.key), true return val.value } From c107a399b7ffef1cdc61ac61ba7982c67e80c5e2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 13 Oct 2022 09:50:30 -0400 Subject: [PATCH 006/100] wip: workin on my docs Signed-off-by: Andrew Mason --- doc/viper/viper.md | 148 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 doc/viper/viper.md diff --git a/doc/viper/viper.md b/doc/viper/viper.md new file mode 100644 index 00000000000..b9651b0e9cb --- /dev/null +++ b/doc/viper/viper.md @@ -0,0 +1,148 @@ +# Vitess Viper Guidelines + +### Preface + +## What is Viper? + +[`viper`][viper] is a configuration-management library for Go programs. +It acts as a registry for configuration values coming from a variety of sources, including: + +- Default values. +- Configuration files (JSON, YAML, TOML, and other formats supported), including optionally watching and live-reloading. +- Environment variables. +- Command-line flags, primarily from `pflag.Flag` types. + +It is used by a wide variety of Go projects, including [hugo][hugo] and [the kubernetes operator][kops]. + +## Common Usage + +Broadly speaking, there's two approaches for Vitess to using viper. + +### Approach 1: Everything in the global registry. + +In this approach, we simply: + +```go +import ( + "github.com/spf13/pflag" + "github.com/spf13/viper" + + // See the section on viperutil for details on this package. + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/vt/servenv" +) + +var myValue = viperutil.NewValue( + "mykey", viper.GetString, + viperutil.WithFlags[string]("myflagname"), + viperutil.WithDefault("defaultvalue"), +) + +func init() { + servenv.OnParseFor("mycmd", func(fs *pflag.FlagSet) { + fs.String("myflagname", myValue.Value(), "help text for myflagname") + myValue.Bind( + /* nil here means use the global viper */ nil, + fs, + ) + }) +} + +func DoAThingWithMyValue() { + fmt.Println("mykey=", myValue.Get()) +} +``` + +Pros: +- Easy to read and write. +- Easy to debug (we can provide a flag to all binaries that results in calling `viper.Debug()`, which would dump out every setting from every vitess module). + +Cons: +- Requires us to be disciplined about cross-module boundaries. + - Anyone anywhere can then do `viper.GetString("mykey")` and retrieve the value. + - Even more scarily, anyone anywhere can _override_ the value via `viper.Set("mykey", 2)` (notice I've even changed the type here). + - Even more _more_ scarily, see below about threadsafety for how dangerous this can be. + +### Approach 2: Package-local vipers + +Instead of putting everything in the global registry, each package can declare a local `viper` instance and put its configuration there. Instead of the above example, this would look like: + +```go +import ( + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/vt/servenv" +) + +var ( + v = viper.New() + myValue = viperutil.NewValue( + "mykey", v.GetString, + viperutil.WithFlags[string]("myflagname"), + viperutil.WithDefault("defaultvalue"), + ), +) + +func init() { + servenv.OnParseFor("mycmd", func(fs *pflag.FlagSet) { + fs.String("myflagname", myValue.Value(), "help text for myflagname") + myValue.Bind( + /* bind to our package-local viper instance */ v, + fs, + ) + }) +} + +func DoAThingWithMyValue() { + fmt.Println("mykey=", myValue.Get()) +} +``` + +Pros: +- Maintains package-private nature of configuration values we currently have. + +Cons: +- No easy "show me all the debug settings everywhere". We would have to have each package expose a function to call, or come up with some other solution. +- Hard to read and write. + - "Wait, what is this `v` thing and where did it come from? + +### Approach 2.2: Readability changes via import alias. + +To address the readability issue in Approach 2, we could alias the viper import to let us write "normal" looking viper code: + +```go +import ( + _viper "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil" +) + +var ( + viper = _viper.New() + myValue = viperutil.NewValue( + "mykey", viper.GetString, + viperutil.WithFlags[string]("myflagname"), + viperutil.WithDefault("defaultvalue"), + ), +) +``` + +The problem _here_ (in addition to it being admittedly a little sneaky), is that if you were to open a new file in the same package, your IDE would very likely pick up the "viper" string and simply import the package without the alias (which also has a `GetString` function), and now you have two files in the same package, one working with the local variable, and the other working with the global registry, and it would be _verrrrry_ tricky to notice in code review. To address that we could write a simple linter to verify that (with the exception of explicitly allow-listed modules), all Vitess modules only ever import `viper` as the aliased version. + +## `go/viperutil` + +## `go/viperutil/viperget` + +## Caveats and Gotchas + +- [ ] case-(in)sensitivity. +- [ ] Threadsafety. +- [ ] `Sub` is split-brain +- [ ] `Unmarshal*` functions rely on `mapstructure` tags, not `json|yaml|...` tags. +- [ ] Any config files/paths added _after_ calling `WatchConfig` will not get picked up. + +[viper]: https://github.com/spf13/viper +[hugo]: https://github.com/gohugoio/hugo +[kops]: https://github.com/kubernetes/kops From 35365b2ed8dc012974dd48e84abf1e29c4b71ddb Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 17 Oct 2022 06:33:22 -0400 Subject: [PATCH 007/100] WIP: config reading Signed-off-by: Andrew Mason --- doc/viper/viper.md | 37 +++++++++ go/flagutil/enum.go | 97 ++++++++++++++++++++++ go/viperutil/config.go | 147 ++++++++++++++++++++++++++++++++++ go/viperutil/viperget/path.go | 21 +++++ go/vt/servenv/servenv.go | 17 ++++ 5 files changed, 319 insertions(+) create mode 100644 go/flagutil/enum.go create mode 100644 go/viperutil/config.go create mode 100644 go/viperutil/viperget/path.go diff --git a/doc/viper/viper.md b/doc/viper/viper.md index b9651b0e9cb..b039f2e1dd4 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -131,6 +131,41 @@ var ( The problem _here_ (in addition to it being admittedly a little sneaky), is that if you were to open a new file in the same package, your IDE would very likely pick up the "viper" string and simply import the package without the alias (which also has a `GetString` function), and now you have two files in the same package, one working with the local variable, and the other working with the global registry, and it would be _verrrrry_ tricky to notice in code review. To address that we could write a simple linter to verify that (with the exception of explicitly allow-listed modules), all Vitess modules only ever import `viper` as the aliased version. +## Config File(s) + +All vitess components will support taking a single, static (i.e. not "watched" via `WatchConfig`) configuration file. + +The file will be loaded via `ReadInConfig`. +We will log a warning if no file was found, but will not abort (see [docs][viper_read_in_config_docs]). + +Subsystems may allow for specifying additional (optionally dynamic, per their discretion, but see Caveats below) config files loaded separately from the main config. +They may choose to follow the global example of not aborting on `viper.ConfigFileNotFoundError` or not, per their discretion (**provided deviation from the norm is captured in that subsystems config flag usage**). + +Flags for all binaries: +- `--config-path` + - Default: `$(pwd)` + - EnvVar: `VT_CONFIG_PATH` (parsed exactly like a `$PATH` style shell variable). + - FlagType: `StringSlice` + - Behavior: Paths for `ReadInConfig` to search. +- `--config-type` (default: "") + - Default: `""` + - EnvVar: `VT_CONFIG_TYPE` + - FlagType: `flagutil.StringEnum` + - Values: everything contained in `viper.SupportedExts`, case-insensitive. + - Behavior: Force viper to use a particular unmarshalling strategy; required if the config file does not have an extension (by default, viper infers the config type from the file extension). +- `--config-name` (default: "vtconfig") + - Default: `"vtconfig"` + - EnvVar: `VT_CONFIG_NAME` + - FlagType: `string` + - Behavior: Instructs `ReadInConfig` to only look in `ConfigPaths` for files named with this name (with any supported extension, unless `ConfigType` is also set, in which case only with that extension). +- `--config-file` + - Default: `""` + - EnvVar: `VT_CONFIG_FILE` + - FlagType: `string` + - Behavior: Instructs `ReadInConfig` to search in `ConfigPaths` for explicitly a file with this name. Takes precedence over `ConfigName`. + +TODO: if we go with Approach 2.1 or Approach 2.2 in the above section, we need to work out a way to propagate the `ReadInConfig` outlined here from the global viper back to each of the package-local vipers. + ## `go/viperutil` ## `go/viperutil/viperget` @@ -144,5 +179,7 @@ The problem _here_ (in addition to it being admittedly a little sneaky), is that - [ ] Any config files/paths added _after_ calling `WatchConfig` will not get picked up. [viper]: https://github.com/spf13/viper +[viper_read_in_config_docs]: https://github.com/spf13/viper#reading-config-files + [hugo]: https://github.com/gohugoio/hugo [kops]: https://github.com/kubernetes/kops diff --git a/go/flagutil/enum.go b/go/flagutil/enum.go new file mode 100644 index 00000000000..64d56e22207 --- /dev/null +++ b/go/flagutil/enum.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flagutil + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +// TODO: docs for all these. +type StringEnum struct { + name string + val string + + caseInsensitive bool + choices map[string]struct{} + choiceNames []string + choiceMapper func(string) string +} + +var ErrInvalidChoice = errors.New("invalid choice for enum") + +func NewStringEnum(name string, initialValue string, choices []string) *StringEnum { + return newStringEnum(name, initialValue, choices, false) +} + +func NewCaseInsensitiveStringEnum(name string, initialValue string, choices []string) *StringEnum { + return newStringEnum(name, initialValue, choices, true) +} + +func newStringEnum(name string, initialValue string, choices []string, caseInsensitive bool) *StringEnum { + choiceMapper := func(s string) string { return s } + choiceMap := map[string]struct{}{} + + if caseInsensitive { + choiceMapper = strings.ToLower + } + + for _, choice := range choices { + choiceMap[choiceMapper(choice)] = struct{}{} + } + + choiceNames := make([]string, 0, len(choiceMap)) + for choice := range choiceMap { + choiceNames = append(choiceNames, choice) + } + sort.Strings(choiceNames) + + if _, ok := choiceMap[choiceMapper(initialValue)]; !ok { + // This will panic if we've misconfigured something in the source code. + // It's not a user-error, so it had damn-well be better caught by a test + // somewhere. + panic("TODO: error message goes here") + } + + return &StringEnum{ + name: name, + val: initialValue, + choices: choiceMap, + choiceNames: choiceNames, + choiceMapper: choiceMapper, + } +} + +func (s *StringEnum) Set(arg string) error { + if _, ok := s.choices[s.choiceMapper(arg)]; !ok { + msg := "%w (valid choices: %v" + if s.caseInsensitive { + msg += " [case insensitive]" + } + msg += ")" + return fmt.Errorf(msg, ErrInvalidChoice, s.choiceNames) + } + + s.val = arg + + return nil +} + +func (s *StringEnum) String() string { return s.val } +func (s *StringEnum) Type() string { return "string" } diff --git a/go/viperutil/config.go b/go/viperutil/config.go new file mode 100644 index 00000000000..721068aeaa8 --- /dev/null +++ b/go/viperutil/config.go @@ -0,0 +1,147 @@ +package viperutil + +import ( + "os" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/flagutil" + "vitess.io/vitess/go/viperutil/viperget" + "vitess.io/vitess/go/vt/log" +) + +var ( + v = viper.New() + configPaths = NewValue( + "config.paths", + viperget.Path(v), + WithEnvVars[[]string]("VT_CONFIG_PATH"), + WithFlags[[]string]("config-path"), + ) + + configType = NewValue( + "config.type", + v.GetString, + WithEnvVars[string]("VT_CONFIG_TYPE"), + WithFlags[string]("config-type"), + ) + _configTypeFlagVal = flagutil.NewCaseInsensitiveStringEnum("config-type", configType.Value(), viper.SupportedExts) + + configName = NewValue( + "config.name", + v.GetString, + WithDefault("vtconfig"), + WithEnvVars[string]("VT_CONFIG_NAME"), + WithFlags[string]("config-name"), + ) + configFile = NewValue( + "config.file", + v.GetString, + WithEnvVars[string]("VT_CONFiG_FILE"), + WithFlags[string]("config-file"), + ) +) + +func init() { + wd, _ := os.Getwd() + configPaths.value = append(configPaths.value, wd) +} + +func RegisterDefaultConfigFlags(fs *pflag.FlagSet) { + fs.StringSlice("config-path", configPaths.Value(), "") // TODO: usage here + configPaths.Bind(v, fs) + + fs.Var(_configTypeFlagVal, "config-type", "") // TODO: usage here + configType.Bind(v, fs) + + fs.String("config-name", configName.Value(), "") // TODO: usage here + configName.Bind(v, fs) + + fs.String("config-file", configFile.Value(), "") // TODO: usage here + configFile.Bind(v, fs) +} + +type ConfigFileNotFoundHandling int + +const ( + IgnoreConfigFileNotFound ConfigFileNotFoundHandling = iota + WarnOnConfigFileNotFound + ExitOnConfigFileNotFound +) + +type ConfigOptions struct { + Paths []string + Type string + Name string + File string +} + +// ReadInConfig attempts to read a config into the given viper instance. +// +// The ConfigOptions govern the config file searching behavior, with Paths, +// Type, Name, and File behaving as described by the viper documentation [1]. +// +// The ConfigFileNotFoundHandling controls whether not finding a config file +// should be ignored, treated as a warning, or exit the program. +// +// [1]: https://pkg.go.dev/github.com/spf13/viper#readme-reading-config-files +func ReadInConfig(v *viper.Viper, opts ConfigOptions, handling ConfigFileNotFoundHandling) error { + for _, path := range opts.Paths { + v.AddConfigPath(path) + } + + if opts.Type != "" { + v.SetConfigType(opts.Type) + } + + if opts.Name != "" { + v.SetConfigName(opts.Name) + } + + if opts.File != "" { + v.SetConfigFile(opts.File) + } + + err := v.ReadInConfig() + if err != nil { + if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { + switch handling { + case IgnoreConfigFileNotFound: + return nil + case WarnOnConfigFileNotFound: + log.Warningf(nferr.Error()) + case ExitOnConfigFileNotFound: + log.Exitf(nferr.Error()) + } + } + } + + return err +} + +// ReadInDefaultConfig reads the default config (governed by the config-* flags +// defined in this package) into the global viper singleton. +// +// It is called by servenv immediately after parsing the command-line flags. +// Modules that manage their own viper instances should merge in this config at +// any point after parsing has occurred (**not** in an init func): +// +// // Assuming this package has initialized a viper as `v := viper.New()`, +// // this merges all settings on the global viper into the local viper. +// v.MergeConfigMap(viper.AllSettings()) +// +// This function will log a warning if a config file cannot be found, since +// users may not necessarily wish to use files to manage their configurations. +func ReadInDefaultConfig() error { + return ReadInConfig( + viper.GetViper(), + ConfigOptions{ + Paths: configPaths.Get(), + Type: configType.Get(), + Name: configName.Get(), + File: configFile.Get(), + }, + WarnOnConfigFileNotFound, + ) +} diff --git a/go/viperutil/viperget/path.go b/go/viperutil/viperget/path.go new file mode 100644 index 00000000000..080de7de0d4 --- /dev/null +++ b/go/viperutil/viperget/path.go @@ -0,0 +1,21 @@ +package viperget + +import ( + "strings" + + "github.com/spf13/viper" +) + +func Path(v *viper.Viper) func(key string) []string { + return func(key string) (paths []string) { + for _, val := range v.GetStringSlice(key) { + if val != "" { + for _, path := range strings.Split(val, ":") { + paths = append(paths, path) + } + } + } + + return paths + } +} diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index e4464020cbf..534bc66887a 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -44,6 +44,7 @@ import ( "vitess.io/vitess/go/netutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/trace" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/grpccommon" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -341,6 +342,8 @@ func ParseFlags(cmd string) { log.Exitf("%s doesn't take any positional arguments, got '%s'", cmd, strings.Join(args, " ")) } + readViperConfig(cmd) + logutil.PurgeLogs() } @@ -371,11 +374,23 @@ func ParseFlagsWithArgs(cmd string) []string { log.Exitf("%s expected at least one positional argument", cmd) } + readViperConfig(cmd) + logutil.PurgeLogs() return args } +func readViperConfig(cmd string) { + if err := viperutil.ReadInDefaultConfig(); err != nil { + log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) + } + + // TODO: MergeConfigMap with servenv's local viper (which does not exist + // yet) before returning. + return +} + // Flag installations for packages that servenv imports. We need to register // here rather than in those packages (which is what we would normally do) // because that would create a dependency cycle. @@ -428,6 +443,8 @@ func init() { OnParse(log.RegisterFlags) // Flags in package logutil are installed for all binaries. OnParse(logutil.RegisterFlags) + // Flags in package viperutil are installed for all binaries. + OnParse(viperutil.RegisterDefaultConfigFlags) } func RegisterFlagsForTopoBinaries(registerFlags func(fs *pflag.FlagSet)) { From 1ad9054925cd6df9297039fbc4b250efe526476b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 17 Oct 2022 10:15:32 -0400 Subject: [PATCH 008/100] lol whoops Signed-off-by: Andrew Mason --- go/flagutil/enum.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/go/flagutil/enum.go b/go/flagutil/enum.go index 64d56e22207..d4b0e9f1f38 100644 --- a/go/flagutil/enum.go +++ b/go/flagutil/enum.go @@ -62,11 +62,13 @@ func newStringEnum(name string, initialValue string, choices []string, caseInsen } sort.Strings(choiceNames) - if _, ok := choiceMap[choiceMapper(initialValue)]; !ok { - // This will panic if we've misconfigured something in the source code. - // It's not a user-error, so it had damn-well be better caught by a test - // somewhere. - panic("TODO: error message goes here") + if initialValue != "" { + if _, ok := choiceMap[choiceMapper(initialValue)]; !ok { + // This will panic if we've misconfigured something in the source code. + // It's not a user-error, so it had damn-well be better caught by a test + // somewhere. + panic("TODO: error message goes here") + } } return &StringEnum{ From 16931c880014037529fca1173b88926d4042bc77 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 18 Oct 2022 09:29:45 -0400 Subject: [PATCH 009/100] wip: vipersync for threadsafe(ish) watched configs Signed-off-by: Andrew Mason --- doc/viper/viper.md | 4 + go/viperutil/vipersync/viper.go | 109 ++++++++++++++++++++ go/viperutil/vipersync/viper_test.go | 145 +++++++++++++++++++++++++++ 3 files changed, 258 insertions(+) create mode 100644 go/viperutil/vipersync/viper.go create mode 100644 go/viperutil/vipersync/viper_test.go diff --git a/doc/viper/viper.md b/doc/viper/viper.md index b039f2e1dd4..422986020ed 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -166,6 +166,10 @@ Flags for all binaries: TODO: if we go with Approach 2.1 or Approach 2.2 in the above section, we need to work out a way to propagate the `ReadInConfig` outlined here from the global viper back to each of the package-local vipers. +### Watching Configs + +TODO: see `go/viperutil/vipersync` + ## `go/viperutil` ## `go/viperutil/viperget` diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go new file mode 100644 index 00000000000..060510100f9 --- /dev/null +++ b/go/viperutil/vipersync/viper.go @@ -0,0 +1,109 @@ +package vipersync + +import ( + "fmt" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil" +) + +// TODO: document all this + +type Viper struct { + disk *viper.Viper + live *viper.Viper + keys map[string]*syncedKey + + watchingConfig bool +} + +type syncedKey struct { + updates chan chan struct{} +} + +func NewViper(v *viper.Viper) *Viper { + sv := &Viper{ + disk: v, + live: viper.New(), + keys: map[string]*syncedKey{}, + } + + // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an + // older version of viper it used to actually handle errors, but now it + // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. + _ = sv.live.MergeConfigMap(sv.disk.AllSettings()) + sv.disk.OnConfigChange(func(in fsnotify.Event) { + chs := make([]chan struct{}, 0, len(sv.keys)) + // Inform each key that an update is coming. + for _, key := range sv.keys { + select { + case <-key.updates: + // The previous update channel never got used by a reader. + // This means that no calls to a Get* func for that key happened + // between the last config change and now. + default: + } + + // Create a new channel for signalling the current config change. + ch := make(chan struct{}) + key.updates <- ch + chs = append(chs, ch) + } + + // Now, every key is blocked for reading; we can atomically swap the + // config on disk for the config in memory. + _ = sv.live.MergeConfigMap(sv.disk.AllSettings()) + + // Unblock each key until the next update. + for _, ch := range chs { + close(ch) + } + }) + + return sv +} + +func (v *Viper) WatchConfig() { + if v.watchingConfig { + panic("WatchConfig called twice on synchronized viper") + } + + v.watchingConfig = true + v.disk.WatchConfig() +} + +func BindValue[T any](v *Viper, value *viperutil.Value[T], fs *pflag.FlagSet) { + value.Bind(v.live, fs) +} + +func AdaptGetter[T any](v *Viper, key string, getter func(v *viper.Viper) T) func(key string) T { + if v.watchingConfig { + panic("cannot adapt getter to synchronized viper which is already watching a config") + } + + if _, ok := v.keys[key]; ok { + panic(fmt.Sprintf("already adapted a getter for key %s", key)) + } + + sk := &syncedKey{ + updates: make(chan chan struct{}, 1), + } + + v.keys[key] = sk + + return func(key string) T { + select { + case update := <-sk.updates: + // There's an update in progress, wait for channel close before + // reading. + <-update + return getter(v.live) + default: + // No ongoing update, read. + return getter(v.live) + } + } +} diff --git a/go/viperutil/vipersync/viper_test.go b/go/viperutil/vipersync/viper_test.go new file mode 100644 index 00000000000..d684d47a089 --- /dev/null +++ b/go/viperutil/vipersync/viper_test.go @@ -0,0 +1,145 @@ +package vipersync_test + +import ( + "context" + "encoding/json" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/vipersync" +) + +func TestWatchConfig(t *testing.T) { + type config struct { + A, B int + } + + tmp, err := os.CreateTemp(".", "TestWatchConfig_*.json") + require.NoError(t, err) + t.Cleanup(func() { os.Remove(tmp.Name()) }) + + stat, err := os.Stat(tmp.Name()) + require.NoError(t, err) + + writeConfig := func(a, b int) error { + data, err := json.Marshal(&config{A: a, B: b}) + if err != nil { + return err + } + + return os.WriteFile(tmp.Name(), data, stat.Mode()) + } + writeRandomConfig := func() error { + a, b := rand.Intn(100), rand.Intn(100) + return writeConfig(a, b) + } + + require.NoError(t, writeRandomConfig()) + + v := viper.New() + v.SetConfigFile(tmp.Name()) + require.NoError(t, v.ReadInConfig()) + + wCh, rCh := make(chan struct{}), make(chan struct{}) + v.OnConfigChange(func(in fsnotify.Event) { + select { + case <-rCh: + return + default: + } + + wCh <- struct{}{} + // block forever to prevent this viper instance from double-updating. + <-rCh + }) + v.WatchConfig() + + // Make sure that basic, unsynchronized WatchConfig is set up before + // beginning the actual test. + a, b := v.GetInt("a"), v.GetInt("b") + require.NoError(t, writeConfig(a+1, b+1)) + <-wCh // wait for the update to finish + + time.Sleep(time.Millisecond * 1000) + require.Equal(t, a+1, v.GetInt("a")) + require.Equal(t, b+1, v.GetInt("b")) + + rCh <- struct{}{} + + // Now, set up our synchronized viper and do a bunch of concurrent reads/writes. + v = viper.New() + v.SetConfigFile(tmp.Name()) + require.NoError(t, v.ReadInConfig()) + + sv := vipersync.NewViper(v) + A := viperutil.NewValue("a", + vipersync.AdaptGetter(sv, "a", func(v *viper.Viper) func(key string) int { + return v.GetInt + }), + ) + B := viperutil.NewValue("b", + vipersync.AdaptGetter(sv, "b", func(v *viper.Viper) func(key string) int { + return v.GetInt + }), + ) + vipersync.BindValue(sv, A, nil) + vipersync.BindValue(sv, B, nil) + + sv.WatchConfig() + + var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + + // Sleep between 25 and 50ms between reads. + readJitter := func() time.Duration { + return time.Duration(jitter(25, 50)) * time.Millisecond + } + // Sleep between 75 and 125ms between writes. + writeJitter := func() time.Duration { + return time.Duration(jitter(75, 125)) * time.Millisecond + } + + for i := 0; i < 2; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + default: + } + + switch i % 2 { + case 0: + A.Fetch() + case 1: + B.Fetch() + } + + time.Sleep(readJitter()) + } + }(i) + } + + for i := 0; i < 100; i++ { + require.NoError(t, writeRandomConfig()) + time.Sleep(writeJitter()) + } + + cancel() + wg.Wait() +} + +func jitter(min, max int) int { + return min + rand.Intn(max-min+1) +} From 714f38f62904c6fa5ccb8343df4d1d164c8c8a64 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 19 Oct 2022 14:27:19 -0400 Subject: [PATCH 010/100] Some threadsafety improvements, not quite there yet Signed-off-by: Andrew Mason --- go/viperutil/viper.go | 55 +++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index f50dee2ef2b..6804e997436 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -61,8 +61,8 @@ type Value[T any] struct { hasDefault bool resolve func(string) T - bound bool - loaded bool + bound chan struct{} + loaded chan struct{} } // NewValue returns a viper-backed value with the given key, lookup function, @@ -71,12 +71,17 @@ func NewValue[T any](key string, getFunc func(string) T, opts ...Option[T]) *Val val := &Value[T]{ key: key, resolve: getFunc, + bound: make(chan struct{}, 1), + loaded: make(chan struct{}, 1), } for _, opt := range opts { opt(val) } + val.bound <- struct{}{} + val.loaded <- struct{}{} + return val } @@ -90,12 +95,20 @@ func NewValue[T any](key string, getFunc func(string) T, opts ...Option[T]) *Val // additional aliases for the value's key. This function panics if a flag name // is specified that is not defined on the flag set. func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { - if val.bound { - return - } + val.tryBind(v, fs) +} - val.bound = true +func (val *Value[T]) tryBind(v *viper.Viper, fs *pflag.FlagSet) { + select { + case _, ok := <-val.bound: + if ok { + val.bind(v, fs) + close(val.bound) + } + } +} +func (val *Value[T]) bind(v *viper.Viper, fs *pflag.FlagSet) { if v == nil { v = viper.GetViper() } @@ -148,16 +161,21 @@ func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { func (val *Value[T]) Fetch() T { // Prior to fetching anything, bind the value to the global viper if it // hasn't been bound to some viper already. - val.Bind(nil, nil) - val.value, val.loaded = val.resolve(val.key), true - return val.value + val.tryBind(nil, nil) + return val.resolve(val.key) } // Get returns the underlying value, loading from the backing viper only on // first use. For dynamic reloading, use Fetch. func (val *Value[T]) Get() T { - if !val.loaded { - val.Fetch() + val.tryBind(nil, nil) + select { + case _, ok := <-val.loaded: + if ok { + // We're the first to load; resolve the key and set the value. + val.value = val.resolve(val.key) + close(val.loaded) + } } return val.value @@ -167,7 +185,20 @@ func (val *Value[T]) Get() T { // previously loaded (either via Fetch or Get), and the value was not created // with a WithDefault option, the return value is whatever the zero value for // the type T is. -func (val *Value[T]) Value() T { +func (val *Value[T]) Value() (value T) { + select { + case _, ok := <-val.loaded: + if ok { + // No one has loaded yet, and no one is loading. + // Return the value passed to us in the constructor, and + // put the sentinel value back so another Get call can proceed + // to actually load from the backing viper. + value = val.value + val.loaded <- struct{}{} + return value + } + } + return val.value } From a53efceba179f86bdf362d9cdddb650e034296ee Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 20 Oct 2022 10:12:17 -0400 Subject: [PATCH 011/100] Add Subscribe/Notify api so live configs don't need to poll for updates Signed-off-by: Andrew Mason --- go/viperutil/vipersync/viper.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index 060510100f9..156731878c4 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -17,6 +17,7 @@ type Viper struct { live *viper.Viper keys map[string]*syncedKey + subscribers []chan<- struct{} watchingConfig bool } @@ -61,6 +62,13 @@ func NewViper(v *viper.Viper) *Viper { for _, ch := range chs { close(ch) } + + for _, ch := range sv.subscribers { + select { + case ch <- struct{}{}: + default: + } + } }) return sv @@ -75,6 +83,14 @@ func (v *Viper) WatchConfig() { v.disk.WatchConfig() } +func (v *Viper) Notify(ch chan<- struct{}) { + if v.watchingConfig { + panic("cannot Notify after starting to watch config") + } + + v.subscribers = append(v.subscribers, ch) +} + func BindValue[T any](v *Viper, value *viperutil.Value[T], fs *pflag.FlagSet) { value.Bind(v.live, fs) } From c13fb667d2ab928dfedf95cca96a9bd5a93cf80c Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 24 Oct 2022 06:17:03 -0400 Subject: [PATCH 012/100] fix adapter signature to what i meant the first time Signed-off-by: Andrew Mason --- go/viperutil/vipersync/viper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index 156731878c4..c543ce1b657 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -95,7 +95,7 @@ func BindValue[T any](v *Viper, value *viperutil.Value[T], fs *pflag.FlagSet) { value.Bind(v.live, fs) } -func AdaptGetter[T any](v *Viper, key string, getter func(v *viper.Viper) T) func(key string) T { +func AdaptGetter[T any](v *Viper, key string, getter func(v *viper.Viper) func(key string) T) func(key string) T { if v.watchingConfig { panic("cannot adapt getter to synchronized viper which is already watching a config") } @@ -116,10 +116,10 @@ func AdaptGetter[T any](v *Viper, key string, getter func(v *viper.Viper) T) fun // There's an update in progress, wait for channel close before // reading. <-update - return getter(v.live) + return getter(v.live)(key) default: // No ongoing update, read. - return getter(v.live) + return getter(v.live)(key) } } } From d56f4aca0bba44c610d517449e5eb24777a58b1f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 24 Oct 2022 06:42:44 -0400 Subject: [PATCH 013/100] begrudgingly come crawling back to an RWMutex implementation Signed-off-by: Andrew Mason --- go/viperutil/vipersync/viper.go | 73 +++++++++++++++++---------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index c543ce1b657..6c11699ba01 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -2,6 +2,7 @@ package vipersync import ( "fmt" + "sync" "github.com/fsnotify/fsnotify" "github.com/spf13/pflag" @@ -15,21 +16,44 @@ import ( type Viper struct { disk *viper.Viper live *viper.Viper - keys map[string]*syncedKey + keys map[string]lockable subscribers []chan<- struct{} watchingConfig bool } -type syncedKey struct { - updates chan chan struct{} +// lockable represents a structure that contains a lockable field. +// +// we need this interface in order to store our map of synced keys in the synced +// viper. unfortunately, we cannot do +// +// for _, key := range sv.keys { +// key.m.Lock() +// defer key.m.Unlock() +// } +// +// because generics don't (yet?) allow a collection of _different_ generic types +// (e.g. you can't have []T{1, "hello", false}, because they are not all the same T), +// so we abstract over the interface. it's not exported because no one outside +// this package should be accessing a synced key's mutex. +type lockable interface { + locker() sync.Locker +} + +type syncedKey[T any] struct { + m sync.RWMutex + get func(key string) T +} + +func (sk *syncedKey[T]) locker() sync.Locker { + return &sk.m } func NewViper(v *viper.Viper) *Viper { sv := &Viper{ disk: v, live: viper.New(), - keys: map[string]*syncedKey{}, + keys: map[string]lockable{}, } // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an @@ -37,32 +61,17 @@ func NewViper(v *viper.Viper) *Viper { // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. _ = sv.live.MergeConfigMap(sv.disk.AllSettings()) sv.disk.OnConfigChange(func(in fsnotify.Event) { - chs := make([]chan struct{}, 0, len(sv.keys)) // Inform each key that an update is coming. for _, key := range sv.keys { - select { - case <-key.updates: - // The previous update channel never got used by a reader. - // This means that no calls to a Get* func for that key happened - // between the last config change and now. - default: - } - - // Create a new channel for signalling the current config change. - ch := make(chan struct{}) - key.updates <- ch - chs = append(chs, ch) + key.locker().Lock() + // This won't fire until after the config has been updated on sv.live. + defer key.locker().Unlock() } - // Now, every key is blocked for reading; we can atomically swap the + // Now, every key is blocked from reading; we can atomically swap the // config on disk for the config in memory. _ = sv.live.MergeConfigMap(sv.disk.AllSettings()) - // Unblock each key until the next update. - for _, ch := range chs { - close(ch) - } - for _, ch := range sv.subscribers { select { case ch <- struct{}{}: @@ -104,22 +113,16 @@ func AdaptGetter[T any](v *Viper, key string, getter func(v *viper.Viper) func(k panic(fmt.Sprintf("already adapted a getter for key %s", key)) } - sk := &syncedKey{ - updates: make(chan chan struct{}, 1), + sk := &syncedKey[T]{ + get: getter(v.live), } v.keys[key] = sk return func(key string) T { - select { - case update := <-sk.updates: - // There's an update in progress, wait for channel close before - // reading. - <-update - return getter(v.live)(key) - default: - // No ongoing update, read. - return getter(v.live)(key) - } + sk.m.RLock() + defer sk.m.RUnlock() + + return sk.get(key) } } From a7f61a33139e22c9d5cda68071e044f38e07b101 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 25 Oct 2022 06:27:53 -0400 Subject: [PATCH 014/100] adjust public Bind method (and only the public version) to allow secondary calls Signed-off-by: Andrew Mason --- go/viperutil/viper.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index 6804e997436..0c392693973 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -94,8 +94,19 @@ func NewValue[T any](key string, getFunc func(string) T, opts ...Option[T]) *Val // flag names do not match the value's canonical key, they are registered as // additional aliases for the value's key. This function panics if a flag name // is specified that is not defined on the flag set. +// +// Bind is not safe to call concurrently with other calls to Bind, Fetch, Get, +// or Value. It is recommended to call Bind at least once before calling those +// other 3 methods; otherwise the default or zero value for T will be returned. func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { - val.tryBind(v, fs) + select { + case _, ok := <-val.bound: + if ok { + defer func() { val.bound <- struct{}{} }() + } + } + + val.bind(v, fs) } func (val *Value[T]) tryBind(v *viper.Viper, fs *pflag.FlagSet) { From ae1eb1c3d5c42132b5e8a6b2f7f1ace6160f9a02 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 25 Oct 2022 06:41:54 -0400 Subject: [PATCH 015/100] add Error handling type for reading in configs Signed-off-by: Andrew Mason --- go/viperutil/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index 721068aeaa8..db099cccb6e 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -67,6 +67,7 @@ type ConfigFileNotFoundHandling int const ( IgnoreConfigFileNotFound ConfigFileNotFoundHandling = iota WarnOnConfigFileNotFound + ErrorOnConfigFileNotFound ExitOnConfigFileNotFound ) @@ -111,6 +112,8 @@ func ReadInConfig(v *viper.Viper, opts ConfigOptions, handling ConfigFileNotFoun return nil case WarnOnConfigFileNotFound: log.Warningf(nferr.Error()) + case ErrorOnConfigFileNotFound: + return err case ExitOnConfigFileNotFound: log.Exitf(nferr.Error()) } From 21f241118d2fab9dc1ecedc6752e5fb582f31fb8 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 25 Oct 2022 06:43:11 -0400 Subject: [PATCH 016/100] change vipersync Watch signature, also ensure we read in the config at least once before returning to the caller Signed-off-by: Andrew Mason --- go/viperutil/vipersync/viper.go | 15 +++++++++++++-- go/viperutil/vipersync/viper_test.go | 4 +--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index 6c11699ba01..b8a26444aa1 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -1,6 +1,7 @@ package vipersync import ( + "errors" "fmt" "sync" @@ -83,13 +84,23 @@ func NewViper(v *viper.Viper) *Viper { return sv } -func (v *Viper) WatchConfig() { +func (v *Viper) Watch(cfg string) error { if v.watchingConfig { - panic("WatchConfig called twice on synchronized viper") + // TODO: declare an exported error and include the filename in wrapped + // error. + return errors.New("duplicate watch") + } + + if err := viperutil.ReadInConfig(v.disk, viperutil.ConfigOptions{ + File: cfg, + }, viperutil.ErrorOnConfigFileNotFound); err != nil { + return err // TODO: wrap error } v.watchingConfig = true + _ = v.live.MergeConfigMap(v.disk.AllSettings()) v.disk.WatchConfig() + return nil } func (v *Viper) Notify(ch chan<- struct{}) { diff --git a/go/viperutil/vipersync/viper_test.go b/go/viperutil/vipersync/viper_test.go index d684d47a089..8c53bb8c8bc 100644 --- a/go/viperutil/vipersync/viper_test.go +++ b/go/viperutil/vipersync/viper_test.go @@ -76,8 +76,6 @@ func TestWatchConfig(t *testing.T) { // Now, set up our synchronized viper and do a bunch of concurrent reads/writes. v = viper.New() - v.SetConfigFile(tmp.Name()) - require.NoError(t, v.ReadInConfig()) sv := vipersync.NewViper(v) A := viperutil.NewValue("a", @@ -93,7 +91,7 @@ func TestWatchConfig(t *testing.T) { vipersync.BindValue(sv, A, nil) vipersync.BindValue(sv, B, nil) - sv.WatchConfig() + require.NoError(t, sv.Watch(tmp.Name())) var wg sync.WaitGroup ctx, cancel := context.WithCancel(context.Background()) From daae99e6dbd743f3b0c87cd3524196bf30782047 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 26 Oct 2022 09:24:17 -0400 Subject: [PATCH 017/100] refactor vipersync.Viper a bit Signed-off-by: Andrew Mason --- go/viperutil/vipersync/viper.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index b8a26444aa1..6dd6fae0e02 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -57,10 +57,6 @@ func NewViper(v *viper.Viper) *Viper { keys: map[string]lockable{}, } - // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an - // older version of viper it used to actually handle errors, but now it - // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. - _ = sv.live.MergeConfigMap(sv.disk.AllSettings()) sv.disk.OnConfigChange(func(in fsnotify.Event) { // Inform each key that an update is coming. for _, key := range sv.keys { @@ -71,7 +67,7 @@ func NewViper(v *viper.Viper) *Viper { // Now, every key is blocked from reading; we can atomically swap the // config on disk for the config in memory. - _ = sv.live.MergeConfigMap(sv.disk.AllSettings()) + sv.loadFromDisk() for _, ch := range sv.subscribers { select { @@ -98,11 +94,19 @@ func (v *Viper) Watch(cfg string) error { } v.watchingConfig = true - _ = v.live.MergeConfigMap(v.disk.AllSettings()) + v.loadFromDisk() v.disk.WatchConfig() + return nil } +func (v *Viper) loadFromDisk() { + // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an + // older version of viper it used to actually handle errors, but now it + // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. + _ = v.live.MergeConfigMap(v.disk.AllSettings()) +} + func (v *Viper) Notify(ch chan<- struct{}) { if v.watchingConfig { panic("cannot Notify after starting to watch config") From b0566d7e5f9f54ac04d4256d3556be64dbbe3f3f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 26 Oct 2022 09:28:45 -0400 Subject: [PATCH 018/100] More refactors to vipersync: - `NewViper`=>`New` - Constructor no longer takes any input. Prevents callers from hanging on to references to anything internal to the vipersync - config-change callback definition moved out of constructor to just before the actual watch starts Signed-off-by: Andrew Mason --- go/viperutil/vipersync/viper.go | 48 +++++++++++++--------------- go/viperutil/vipersync/viper_test.go | 2 +- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index 6dd6fae0e02..d2f193f7c58 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -50,34 +50,12 @@ func (sk *syncedKey[T]) locker() sync.Locker { return &sk.m } -func NewViper(v *viper.Viper) *Viper { - sv := &Viper{ - disk: v, +func New() *Viper { + return &Viper{ + disk: viper.New(), live: viper.New(), keys: map[string]lockable{}, } - - sv.disk.OnConfigChange(func(in fsnotify.Event) { - // Inform each key that an update is coming. - for _, key := range sv.keys { - key.locker().Lock() - // This won't fire until after the config has been updated on sv.live. - defer key.locker().Unlock() - } - - // Now, every key is blocked from reading; we can atomically swap the - // config on disk for the config in memory. - sv.loadFromDisk() - - for _, ch := range sv.subscribers { - select { - case ch <- struct{}{}: - default: - } - } - }) - - return sv } func (v *Viper) Watch(cfg string) error { @@ -95,6 +73,26 @@ func (v *Viper) Watch(cfg string) error { v.watchingConfig = true v.loadFromDisk() + + v.disk.OnConfigChange(func(in fsnotify.Event) { + // Inform each key that an update is coming. + for _, key := range v.keys { + key.locker().Lock() + // This won't fire until after the config has been updated on v.live. + defer key.locker().Unlock() + } + + // Now, every key is blocked from reading; we can atomically swap the + // config on disk for the config in memory. + v.loadFromDisk() + + for _, ch := range v.subscribers { + select { + case ch <- struct{}{}: + default: + } + } + }) v.disk.WatchConfig() return nil diff --git a/go/viperutil/vipersync/viper_test.go b/go/viperutil/vipersync/viper_test.go index 8c53bb8c8bc..e82bd5d08f9 100644 --- a/go/viperutil/vipersync/viper_test.go +++ b/go/viperutil/vipersync/viper_test.go @@ -77,7 +77,7 @@ func TestWatchConfig(t *testing.T) { // Now, set up our synchronized viper and do a bunch of concurrent reads/writes. v = viper.New() - sv := vipersync.NewViper(v) + sv := vipersync.New() A := viperutil.NewValue("a", vipersync.AdaptGetter(sv, "a", func(v *viper.Viper) func(key string) int { return v.GetInt From c6db9437505eaa96d93eab5c9b150e9fa56304d4 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 31 Oct 2022 10:04:22 -0400 Subject: [PATCH 019/100] WIP WIP WIP checkpointing before i blow away this api for a new version Signed-off-by: Andrew Mason --- go/viperutil/config.go | 39 ++++++++++++ go/viperutil/viper.go | 102 +++++++++++++++++++++++++++++++- go/viperutil/vipersync/viper.go | 1 + go/vt/topo/locks.go | 53 +++++++++++++++++ 4 files changed, 193 insertions(+), 2 deletions(-) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index db099cccb6e..23a4b4b49e2 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -3,6 +3,7 @@ package viperutil import ( "os" + jww "github.com/spf13/jwalterweatherman" "github.com/spf13/pflag" "github.com/spf13/viper" @@ -71,6 +72,21 @@ const ( ExitOnConfigFileNotFound ) +func (handling ConfigFileNotFoundHandling) String() string { + switch handling { + case IgnoreConfigFileNotFound: + return "ignore" + case WarnOnConfigFileNotFound: + return "warn" + case ErrorOnConfigFileNotFound: + return "error" + case ExitOnConfigFileNotFound: + return "exit" + default: + return "" + } +} + type ConfigOptions struct { Paths []string Type string @@ -104,17 +120,25 @@ func ReadInConfig(v *viper.Viper, opts ConfigOptions, handling ConfigFileNotFoun v.SetConfigFile(opts.File) } + jww.DEBUG.Printf("Reading in config with options: %+v; FileNotFoundHandling: %s\n", opts, handling.String()) + err := v.ReadInConfig() if err != nil { + format, args := "Failed to read in config %s: %s", []any{v.ConfigFileUsed(), err.Error()} + if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { switch handling { case IgnoreConfigFileNotFound: + DEBUG(format, args...) return nil case WarnOnConfigFileNotFound: + WARN(format, args...) log.Warningf(nferr.Error()) case ErrorOnConfigFileNotFound: + ERROR(format, args...) return err case ExitOnConfigFileNotFound: + CRITICAL(format, args...) log.Exitf(nferr.Error()) } } @@ -123,6 +147,21 @@ func ReadInConfig(v *viper.Viper, opts ConfigOptions, handling ConfigFileNotFoun return err } +var ( + jwwlog = func(printer interface { + Printf(format string, args ...any) + }) func(format string, args ...any) { + return printer.Printf + } + + TRACE = jwwlog(jww.TRACE) + DEBUG = jwwlog(jww.DEBUG) + INFO = jwwlog(jww.INFO) + WARN = jwwlog(jww.WARN) + ERROR = jwwlog(jww.ERROR) + CRITICAL = jwwlog(jww.CRITICAL) +) + // ReadInDefaultConfig reads the default config (governed by the config-* flags // defined in this package) into the global viper singleton. // diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index 0c392693973..6b99cca13f4 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -14,8 +14,106 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package viperutil provides additional functions for Vitess's use of viper for -// managing configuration of various components. +/* +Package viperutil provides additional functions for Vitess's use of viper for +managing configuration of various components. + +Example usage, without dynamic reload: + + var ( + tracingServer = viperutil.NewValue( + "trace.service", + viper.GetString, + viperutil.WithFlags[string]("tracer"), + viperutil.Withdefault("noop"), + ) + ) + + func init() { + servenv.OnParse(func(fs *pflag.FlagSet) { + fs.String("tracer", tracingServer.Value(), "tracing service to use") + tracingServer.Bind(nil, fs) + }) + } + + func StartTracing(serviceName string) io.Closer { + backend := tracingServer.Get() + factory, ok := tracingBackendFactories[backend] + ... + } + +Example usage, with dynamic reload: + + var ( + syncedViper = vipersync.New() + syncedGetDuration = vipersync.AdaptGetter(syncedViper, func(v *viper.Viper) func(key string) time.Duration { + return v.GetDuration + }) + syncedGetInt = vipersync.AdaptGetter(syncedViper, func(v *viper.Viper) func(key string) int { + return v.GetInt + }) + + // Convenience to remove need to repeat this module's prefix constantly. + keyPrefix = viperutil.KeyPrefix("grpc.client") + + keepaliveTime = viperutil.NewValue( + keyPrefix("keepalive.time"), + syncedGetDuration, + viperutil.WithFlags[time.Duration]("grpc_keepalive_time"), + viperutil.WithDefault(10 * time.Second), + ) + keepaliveTimeout = viperutil.NewValue( + keyPrefix("keepalive.timeout"), + syncedGetDuration, + viperutil.WithFlags[time.Duration]("grpc_keepalive_timeout"), + viperutil.WithDefault(10 * time.Second), + ) + + initialConnWindowSize = viperutil.NewValue( + keyPrefix("initial_conn_window_size"), + syncedGetInt, + viperutil.WithFlags[int]("grpc_initial_conn_window_size"), + ) + ) + + func init() { + binaries := []string{...} + for _, cmd := range binaries { + servenv.OnParseFor(cmd, func(fs *pflag.FlagSet) { + + }) + } + + servenv.OnConfigLoad(func() { + // Watch the config file for changes to our synced variables. + syncedViper.Watch(viper.ConfigFileUsed()) + }) + } + + func DialContext(ctx context.Context, target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + ... + + // Load our dynamic variables, thread-safely. + kaTime := keepaliveTime.Fetch() + kaTimeout := keepaliveTimeout.Fetch() + if kaTime != 0 || kaTimeout != 0 { + kp := keepalive.ClientParameters{ + Time: kaTime, + Timeout: kaTimeout, + PermitWithoutStream: true, + } + newopts = append(newopts, grpc.WithKeepaliveParams(kp)) + } + + if size := initialConnWindowSize.Fetch(); size != 0 { + newopts = append(newopts, grpc.WithInitialConnWindowSize(int32(size))) + } + + // and so on ... + + return grpc.DialContext(ctx, target, newopts...) + } +*/ package viperutil import ( diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go index d2f193f7c58..c13b47eb809 100644 --- a/go/viperutil/vipersync/viper.go +++ b/go/viperutil/vipersync/viper.go @@ -13,6 +13,7 @@ import ( ) // TODO: document all this +// TODO: use jww for logging within go/viperutil type Viper struct { disk *viper.Viper diff --git a/go/vt/topo/locks.go b/go/vt/topo/locks.go index 8d30d85e891..7073b970fe0 100644 --- a/go/vt/topo/locks.go +++ b/go/vt/topo/locks.go @@ -26,9 +26,12 @@ import ( "time" "github.com/spf13/pflag" + "github.com/spf13/viper" _flag "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/trace" + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/vipersync" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/servenv" @@ -43,6 +46,21 @@ var ( // shard / keyspace lock can be acquired for. LockTimeout = 45 * time.Second + sv = vipersync.New() + __getRemoteOperationTimeout = vipersync.AdaptGetter(sv, "topo.remote_operation_timeout", func(v *viper.Viper) func(key string) time.Duration { + return v.GetDuration + }) + remoteOperationTimeout = viperutil.NewValue( + "topo.remote_operation_timeout", + + __getRemoteOperationTimeout, + // vipersync.AdaptGetter(sv, "topo.remote_operation_timeout", func(v *viper.Viper) func(key string) time.Duration { + // return v.GetDuration + // }), + viperutil.WithDefault(30*time.Second), + viperutil.WithFlags[time.Duration]("remote_operation_timeout"), + ) + // RemoteOperationTimeout is used for operations where we have to // call out to another process. // Used for RPC calls (including topo server calls) @@ -71,6 +89,41 @@ func init() { func registerTopoLockFlags(fs *pflag.FlagSet) { fs.DurationVar(&RemoteOperationTimeout, "remote_operation_timeout", RemoteOperationTimeout, "time to wait for a remote operation") fs.DurationVar(&LockTimeout, "lock-timeout", LockTimeout, "Maximum time for which a shard/keyspace lock can be acquired for") + + remoteOperationTimeout.Bind(nil, fs) + + servenv.OnRun(func() { + log.Errorf("servenv.OnRun hook for topo called") + vipersync.BindValue(sv, remoteOperationTimeout, fs) + + cfg := viper.ConfigFileUsed() + if cfg == "" { + log.Errorf("no config file set on global viper") + return + } + + ch := make(chan struct{}, 1) + go func() { + for range ch { + log.Errorf("topo.RemoteOperationTimeout updated; new value: %v", remoteOperationTimeout.Fetch()) + } + + log.Errorf("topo settings channel closed") + }() + + servenv.OnTerm(func() { + close(ch) + }) + + sv.Notify(ch) + if err := sv.Watch(viper.ConfigFileUsed()); err != nil { + log.Errorf("failed to read config in vipersync.disk: %s", err.Error()) + return + } + + log.Errorf("initialized config watcher for topo settings") + log.Errorf("initial topo.RemoteOperationTimeout: %v", remoteOperationTimeout.Fetch()) + }) } // newLock creates a new Lock. From 601c62d7307dc6a0c9f2dd2594b4353bc45f070d Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Nov 2022 12:11:20 -0500 Subject: [PATCH 020/100] wip on v2 of viperutil api Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 210 ++++++++++++++++++ go/viperutil/v2/config_test.go | 31 +++ go/viperutil/v2/debug/debug.go | 14 ++ go/viperutil/v2/funcs/decode.go | 42 ++++ go/viperutil/v2/funcs/get.go | 21 ++ go/viperutil/v2/get_func.go | 165 ++++++++++++++ go/viperutil/v2/internal/log/log.go | 30 +++ go/viperutil/v2/internal/registry/registry.go | 23 ++ go/viperutil/v2/internal/sync/sync.go | 120 ++++++++++ go/viperutil/v2/internal/value/value.go | 135 +++++++++++ go/viperutil/v2/value.go | 22 ++ go/viperutil/v2/viper.go | 61 +++++ go/viperutil/v2/viper_test.go | 20 ++ go/vt/servenv/servenv.go | 31 ++- 14 files changed, 907 insertions(+), 18 deletions(-) create mode 100644 go/viperutil/v2/config.go create mode 100644 go/viperutil/v2/config_test.go create mode 100644 go/viperutil/v2/debug/debug.go create mode 100644 go/viperutil/v2/funcs/decode.go create mode 100644 go/viperutil/v2/funcs/get.go create mode 100644 go/viperutil/v2/get_func.go create mode 100644 go/viperutil/v2/internal/log/log.go create mode 100644 go/viperutil/v2/internal/registry/registry.go create mode 100644 go/viperutil/v2/internal/sync/sync.go create mode 100644 go/viperutil/v2/internal/value/value.go create mode 100644 go/viperutil/v2/value.go create mode 100644 go/viperutil/v2/viper.go create mode 100644 go/viperutil/v2/viper_test.go diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go new file mode 100644 index 00000000000..4dbf6f2ceed --- /dev/null +++ b/go/viperutil/v2/config.go @@ -0,0 +1,210 @@ +package viperutil + +import ( + "fmt" + "os" + "reflect" + "sort" + "strings" + + "github.com/mitchellh/mapstructure" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/v2/funcs" + "vitess.io/vitess/go/viperutil/v2/internal/log" + "vitess.io/vitess/go/viperutil/v2/internal/registry" + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +var ( + configPaths = Configure( + "config.paths", + Options[[]string]{ + GetFunc: funcs.GetPath, + EnvVars: []string{"VT_CONFIG_PATH"}, + FlagName: "config-path", + }, + ) + configType = Configure( + "config.type", + Options[string]{ + EnvVars: []string{"VT_CONFIG_TYPE"}, + FlagName: "config-type", + }, + ) + configName = Configure( + "config.name", + Options[string]{ + Default: "vtconfig", + EnvVars: []string{"VT_CONFIG_NAME"}, + FlagName: "config-name", + }, + ) + configFile = Configure( + "config.file", + Options[string]{ + EnvVars: []string{"VT_CONFIG_FILE"}, + FlagName: "config-file", + }, + ) + configFileNotFoundHandling = Configure( + "config.notfound.handling", + Options[ConfigFileNotFoundHandling]{ + GetFunc: getHandlingValue, + }, + ) +) + +func init() { + wd, err := os.Getwd() + if err != nil { + log.WARN("failed to get working directory (err=%s), not appending to default config-paths", err) + return + } + + configPaths.(*value.Static[[]string]).DefaultVal = []string{wd} +} + +func RegisterFlags(fs *pflag.FlagSet) { + fs.StringSlice("config-path", configPaths.Default(), "Paths to search for config files in.") + fs.String("config-type", configType.Default(), "Config file type (omit to infer config type from file extension).") + fs.String("config-name", configName.Default(), "Name of the config file (without extension) to search for.") + fs.String("config-file", configFile.Default(), "Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored.") + + var h = WarnOnConfigFileNotFound + fs.Var(&h, "config-file-not-found-handling", fmt.Sprintf("Behavior when a config file is not found. (Options: %s)", strings.Join(handlingNames, ", "))) + + BindFlags(fs, configPaths, configType, configName, configFile) +} + +func LoadConfig() error { + var err error + switch file := configFile.Get(); file { + case "": + if name := configName.Get(); name != "" { + registry.Static.SetConfigName(name) + + for _, path := range configPaths.Get() { + registry.Static.AddConfigPath(path) + } + + if cfgType := configType.Get(); cfgType != "" { + registry.Static.SetConfigType(cfgType) + } + + err = registry.Static.ReadInConfig() + } + default: + registry.Static.SetConfigFile(file) + err = registry.Static.ReadInConfig() + } + + if err != nil { + if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { + switch configFileNotFoundHandling.Get() { + case IgnoreConfigFileNotFound: + // TODO: finish this switch block. + } + log.WARN("Failed to read in config %s: %s", registry.Static.ConfigFileUsed(), nferr.Error()) + err = nil + } + } + + if err != nil { + return err + } + + return registry.Dynamic.Watch(registry.Static) +} + +func NotifyConfigReload(ch chan<- struct{}) { + registry.Dynamic.Notify(ch) +} + +type ConfigFileNotFoundHandling int + +const ( + IgnoreConfigFileNotFound ConfigFileNotFoundHandling = iota + WarnOnConfigFileNotFound + ErrorOnConfigFileNotFound + ExitOnConfigFileNotFound +) + +var ( + handlingNames []string + handlingNamesToValues = map[string]int{ + "ignore": int(IgnoreConfigFileNotFound), + "warn": int(WarnOnConfigFileNotFound), + "error": int(ErrorOnConfigFileNotFound), + "exit": int(ExitOnConfigFileNotFound), + } + handlingValuesToNames map[int]string +) + +func getHandlingValue(v *viper.Viper) func(key string) ConfigFileNotFoundHandling { + return func(key string) (h ConfigFileNotFoundHandling) { + if err := v.UnmarshalKey(key, &h, viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(decodeHandlingValue))); err != nil { + h = IgnoreConfigFileNotFound + log.WARN("failed to unmarshal %s: %s; defaulting to %s", err.Error(), key, err.Error(), h.String()) + } + + return h + } +} + +func decodeHandlingValue(from, to reflect.Type, data any) (any, error) { + var h ConfigFileNotFoundHandling + if to != reflect.TypeOf(h) { + return data, nil + } + + switch { + case from.ConvertibleTo(reflect.TypeOf(h)): + return data.(ConfigFileNotFoundHandling), nil + case from.Kind() == reflect.Int: + return ConfigFileNotFoundHandling(data.(int)), nil + case from.Kind() == reflect.String: + if err := h.Set(data.(string)); err != nil { + return h, err + } + + return h, nil + } + + return data, fmt.Errorf("invalid value for ConfigHandlingType: %v", data) +} + +func init() { + handlingNames = make([]string, 0, len(handlingNamesToValues)) + handlingValuesToNames = make(map[int]string, len(handlingNamesToValues)) + + for name, val := range handlingNamesToValues { + handlingValuesToNames[val] = name + handlingNames = append(handlingNames, name) + } + + sort.Slice(handlingNames, func(i, j int) bool { + return handlingNames[i] < handlingNames[j] + }) +} + +func (h *ConfigFileNotFoundHandling) Set(arg string) error { + larg := strings.ToLower(arg) + if v, ok := handlingNamesToValues[larg]; ok { + *h = ConfigFileNotFoundHandling(v) + return nil + } + + return fmt.Errorf("unknown handling name %s", arg) +} + +func (h *ConfigFileNotFoundHandling) String() string { + if name, ok := handlingValuesToNames[int(*h)]; ok { + return name + } + + return "" +} + +func (h *ConfigFileNotFoundHandling) Type() string { return "ConfigFileNotFoundHandling" } diff --git a/go/viperutil/v2/config_test.go b/go/viperutil/v2/config_test.go new file mode 100644 index 00000000000..151a9f05b3e --- /dev/null +++ b/go/viperutil/v2/config_test.go @@ -0,0 +1,31 @@ +package viperutil + +import ( + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetConfigHandlingValue(t *testing.T) { + v := viper.New() + v.SetConfigType("yaml") + + cfg := ` +foo: 2 +bar: "2" # not valid, defaults to "ignore" (0) +baz: error +duration: 10h +` + err := v.ReadConfig(strings.NewReader(strings.NewReplacer("\t", " ").Replace(cfg))) + require.NoError(t, err) + + getHandlingValueFunc := getHandlingValue(v) + assert.Equal(t, ErrorOnConfigFileNotFound, getHandlingValueFunc("foo"), "failed to get int value") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("bar"), "failed to get int-like string value") + assert.Equal(t, ErrorOnConfigFileNotFound, getHandlingValueFunc("baz"), "failed to get string value") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("notset"), "failed to get value on unset key") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("duration"), "failed to get value on duration value") +} diff --git a/go/viperutil/v2/debug/debug.go b/go/viperutil/v2/debug/debug.go new file mode 100644 index 00000000000..47ae495f35b --- /dev/null +++ b/go/viperutil/v2/debug/debug.go @@ -0,0 +1,14 @@ +package debug + +import ( + "github.com/spf13/viper" + "vitess.io/vitess/go/viperutil/v2/internal/registry" +) + +func Debug() { + v := viper.New() + _ = v.MergeConfigMap(registry.Static.AllSettings()) + _ = v.MergeConfigMap(registry.Dynamic.AllSettings()) + + v.Debug() +} diff --git a/go/viperutil/v2/funcs/decode.go b/go/viperutil/v2/funcs/decode.go new file mode 100644 index 00000000000..1816655635c --- /dev/null +++ b/go/viperutil/v2/funcs/decode.go @@ -0,0 +1,42 @@ +package funcs + +// TODO: this creates an import cycle ... IMO tabletenv should not define Seconds, +// it should be in something more akin to flagutil with the other values like +// TabletTypeFlag and friends. + +// import ( +// "reflect" +// "time" +// +// "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" +// ) +// +// func DecodeSeconds(from, to reflect.Type, data any) (any, error) { +// if to != reflect.TypeOf(tabletenv.Seconds(0)) { +// return data, nil +// } +// +// var ( +// n float64 +// ok bool +// ) +// switch from.Kind() { +// case reflect.Float32: +// data := data.(float32) +// n, ok = float64(data), true +// case reflect.Float64: +// n, ok = data.(float64), true +// } +// +// if ok { +// return tabletenv.Seconds(n), nil +// } +// +// if from != reflect.TypeOf(time.Duration(0)) { +// return data, nil +// } +// +// s := tabletenv.Seconds(0) +// s.Set(data.(time.Duration)) +// return s, nil +// } diff --git a/go/viperutil/v2/funcs/get.go b/go/viperutil/v2/funcs/get.go new file mode 100644 index 00000000000..464a5ccad73 --- /dev/null +++ b/go/viperutil/v2/funcs/get.go @@ -0,0 +1,21 @@ +package funcs + +import ( + "strings" + + "github.com/spf13/viper" +) + +func GetPath(v *viper.Viper) func(key string) []string { + return func(key string) (paths []string) { + for _, val := range v.GetStringSlice(key) { + if val != "" { + for _, path := range strings.Split(val, ":") { + paths = append(paths, path) + } + } + } + + return paths + } +} diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go new file mode 100644 index 00000000000..7d4cd1c6023 --- /dev/null +++ b/go/viperutil/v2/get_func.go @@ -0,0 +1,165 @@ +package viperutil + +import ( + "fmt" + "reflect" + "time" + + "github.com/spf13/viper" +) + +func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { + var ( + t T + f any + ) + + typ := reflect.TypeOf(t) + switch typ.Kind() { + case reflect.Bool: + f = func(v *viper.Viper) func(key string) bool { + return v.GetBool + } + case reflect.Int: + f = func(v *viper.Viper) func(key string) int { + return v.GetInt + } + case reflect.Int8: + f = getCastedInt[int8]() + case reflect.Int16: + f = getCastedInt[int16]() + case reflect.Int32: + f = func(v *viper.Viper) func(key string) int32 { + return v.GetInt32 + } + case reflect.Int64: + switch typ { + case reflect.TypeOf(time.Duration(0)): + f = func(v *viper.Viper) func(key string) time.Duration { + return v.GetDuration + } + default: + f = func(v *viper.Viper) func(key string) int64 { + return v.GetInt64 + } + } + case reflect.Uint: + f = func(v *viper.Viper) func(key string) uint { + return v.GetUint + } + case reflect.Uint8: + f = getCastedUint[uint8]() + case reflect.Uint16: + f = getCastedUint[uint16]() + case reflect.Uint32: + f = func(v *viper.Viper) func(key string) uint32 { + return v.GetUint32 + } + case reflect.Uint64: + f = func(v *viper.Viper) func(key string) uint64 { + return v.GetUint64 + } + case reflect.Uintptr: + case reflect.Float32: + f = func(v *viper.Viper) func(key string) float32 { + return func(key string) float32 { + return float32(v.GetFloat64(key)) + } + } + case reflect.Float64: + f = func(v *viper.Viper) func(key string) float64 { + return v.GetFloat64 + } + case reflect.Complex64: + case reflect.Complex128: + case reflect.Array: + case reflect.Chan: + case reflect.Func: + case reflect.Interface: + // TODO: unwrap to see if struct-ish + case reflect.Map: + switch typ.Key().Kind() { + case reflect.String: + switch val := typ.Elem(); val.Kind() { + case reflect.String: + f = func(v *viper.Viper) func(key string) map[string]string { + return v.GetStringMapString + } + case reflect.Slice: + switch val.Elem().Kind() { + case reflect.String: + f = func(v *viper.Viper) func(key string) map[string][]string { + return v.GetStringMapStringSlice + } + } + case reflect.Interface: + f = func(v *viper.Viper) func(key string) map[string]interface{} { + return v.GetStringMap + } + } + } + case reflect.Pointer: + // TODO: unwrap to see if struct-ish + case reflect.Slice: + switch typ.Elem().Kind() { + case reflect.Int: + f = func(v *viper.Viper) func(key string) []int { + return v.GetIntSlice + } + case reflect.String: + f = func(v *viper.Viper) func(key string) []string { + return v.GetStringSlice + } + } + case reflect.String: + f = func(v *viper.Viper) func(key string) string { + return v.GetString + } + case reflect.Struct: + switch typ { + case reflect.TypeOf(time.Time{}): + f = func(v *viper.Viper) func(key string) time.Time { + return v.GetTime + } + default: + f2 := unmarshalFunc[*T]() + f = func(v *viper.Viper) func(key string) T { + getPointer := f2(v) + return func(key string) T { + return *(getPointer(key)) + } + } + } + } + + if f == nil { + panic(fmt.Sprintf("no default GetFunc for type %T; call Configure with a custom GetFunc", t)) + } + return f.(func(v *viper.Viper) func(key string) T) +} + +func unmarshalFunc[T any]() func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + var t T + return func(key string) T { + _ = v.UnmarshalKey(key, t) + return t + } + } +} + +func getCastedInt[T int8 | int16]() func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + return T(v.GetInt(key)) + } + } +} + +func getCastedUint[T uint8 | uint16]() func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + return T(v.GetUint(key)) + } + } +} diff --git a/go/viperutil/v2/internal/log/log.go b/go/viperutil/v2/internal/log/log.go new file mode 100644 index 00000000000..4484a63fd2f --- /dev/null +++ b/go/viperutil/v2/internal/log/log.go @@ -0,0 +1,30 @@ +package log + +import ( + jww "github.com/spf13/jwalterweatherman" + + "vitess.io/vitess/go/vt/log" +) + +var ( + jwwlog = func(printer interface { + Printf(format string, args ...any) + }, vtlogger func(format string, args ...any)) func(format string, args ...any) { + switch vtlogger { + case nil: + return printer.Printf + default: + return func(format string, args ...any) { + printer.Printf(format, args...) + vtlogger(format, args...) + } + } + } + + TRACE = jwwlog(jww.TRACE, nil) + DEBUG = jwwlog(jww.DEBUG, nil) + INFO = jwwlog(jww.INFO, log.Infof) + WARN = jwwlog(jww.WARN, log.Warningf) + ERROR = jwwlog(jww.ERROR, log.Errorf) + CRITICAL = jwwlog(jww.CRITICAL, log.Fatalf) +) diff --git a/go/viperutil/v2/internal/registry/registry.go b/go/viperutil/v2/internal/registry/registry.go new file mode 100644 index 00000000000..0efc7e3ce6c --- /dev/null +++ b/go/viperutil/v2/internal/registry/registry.go @@ -0,0 +1,23 @@ +package registry + +import ( + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/v2/internal/sync" +) + +var ( + Static = viper.New() + Dynamic = sync.New() + + _ Bindable = (*viper.Viper)(nil) + _ Bindable = (*sync.Viper)(nil) +) + +type Bindable interface { + BindEnv(vars ...string) error + BindPFlag(key string, flag *pflag.Flag) error + RegisterAlias(alias string, key string) + SetDefault(key string, value any) +} diff --git a/go/viperutil/v2/internal/sync/sync.go b/go/viperutil/v2/internal/sync/sync.go new file mode 100644 index 00000000000..6409f43bf98 --- /dev/null +++ b/go/viperutil/v2/internal/sync/sync.go @@ -0,0 +1,120 @@ +package sync + +import ( + "errors" + "fmt" + "sync" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +type Viper struct { + m sync.Mutex // prevents races between loadFromDisk and AllSettings + disk *viper.Viper + live *viper.Viper + keys map[string]*sync.RWMutex + + subscribers []chan<- struct{} + watchingConfig bool +} + +func New() *Viper { + return &Viper{ + disk: viper.New(), + live: viper.New(), + keys: map[string]*sync.RWMutex{}, + } +} + +func (v *Viper) Watch(static *viper.Viper) error { + if v.watchingConfig { + // TODO: declare an exported error and include the filename in wrapped + // error. + return errors.New("duplicate watch") + } + + v.disk.SetConfigFile(static.ConfigFileUsed()) + if err := v.disk.ReadInConfig(); err != nil { + return err + } + + v.watchingConfig = true + v.loadFromDisk() + v.disk.OnConfigChange(func(in fsnotify.Event) { + for _, m := range v.keys { + m.Lock() + // This won't fire until after the config has been updated on v.live. + defer m.Unlock() + } + + v.loadFromDisk() + + for _, ch := range v.subscribers { + select { + case ch <- struct{}{}: + default: + } + } + }) + v.disk.WatchConfig() + + return nil +} + +func (v *Viper) Notify(ch chan<- struct{}) { + if v.watchingConfig { + panic("cannot Notify after starting to watch a config") + } + + v.subscribers = append(v.subscribers, ch) +} + +func (v *Viper) AllSettings() map[string]any { + v.m.Lock() + defer v.m.Unlock() + + return v.live.AllSettings() +} + +func (v *Viper) loadFromDisk() { + v.m.Lock() + defer v.m.Unlock() + + // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an + // older version of viper it used to actually handle errors, but now it + // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. + _ = v.live.MergeConfigMap(v.disk.AllSettings()) +} + +// begin implementation of registry.Bindable for sync.Viper + +func (v *Viper) BindEnv(vars ...string) error { return v.live.BindEnv(vars...) } +func (v *Viper) BindPFlag(key string, flag *pflag.Flag) error { return v.live.BindPFlag(key, flag) } +func (v *Viper) RegisterAlias(alias string, key string) { v.live.RegisterAlias(alias, key) } +func (v *Viper) SetDefault(key string, value any) { v.live.SetDefault(key, value) } + +// end implementation of registry.Bindable for sync.Viper + +func AdaptGetter[T any](key string, getter func(v *viper.Viper) func(key string) T, v *Viper) func(key string) T { + if v.watchingConfig { + panic("cannot adapt getter to synchronized viper which is already watching a config") + } + + if _, ok := v.keys[key]; ok { + panic(fmt.Sprintf("already adapted a getter for key %s", key)) + } + + var m sync.RWMutex + v.keys[key] = &m + + boundGet := getter(v.live) + + return func(key string) T { + m.RLock() + defer m.RUnlock() + + return boundGet(key) + } +} diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/v2/internal/value/value.go new file mode 100644 index 00000000000..2b60ed63b0f --- /dev/null +++ b/go/viperutil/v2/internal/value/value.go @@ -0,0 +1,135 @@ +package value + +import ( + "errors" + "fmt" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/v2/internal/registry" + "vitess.io/vitess/go/viperutil/v2/internal/sync" +) + +type Registerable interface { + Key() string + Registry() registry.Bindable + Flag(fs *pflag.FlagSet) (*pflag.Flag, error) +} + +type Base[T any] struct { + KeyName string + DefaultVal T + + GetFunc func(v *viper.Viper) func(key string) T + get func(key string) T + + Aliases []string + FlagName string + EnvVars []string +} + +func (val *Base[T]) Key() string { return val.KeyName } +func (val *Base[T]) Default() T { return val.DefaultVal } +func (val *Base[T]) Get() T { return val.get(val.Key()) } + +var errNoFlagDefined = errors.New("flag not defined") + +func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { + if val.FlagName == "" { + return nil, nil + } + + flag := fs.Lookup(val.FlagName) + if flag == nil { + return nil, fmt.Errorf("%w with name %s (for key %s)", errNoFlagDefined, val.FlagName, val.Key()) // TODO: export error + } + + return flag, nil +} + +func (val *Base[T]) Bind(v registry.Bindable) { + v.SetDefault(val.Key(), val.DefaultVal) + + for _, alias := range val.Aliases { + v.RegisterAlias(alias, val.Key()) + } + + if len(val.EnvVars) > 0 { + vars := append([]string{val.Key()}, val.EnvVars...) + _ = v.BindEnv(vars...) + } +} + +func BindFlags(fs *pflag.FlagSet, values ...Registerable) { + for _, val := range values { + flag, err := val.Flag(fs) + switch { + case err != nil: + // TODO: panic + panic(fmt.Errorf("failed to load flag for %s: %w", val.Key(), err)) + case flag == nil: + continue + } + + _ = val.Registry().BindPFlag(val.Key(), flag) + if flag.Name != val.Key() { + val.Registry().RegisterAlias(flag.Name, val.Key()) + } + } +} + +// func xxx_DELETE_ME_bindFlag(v registry.Bindable, key string, flagName string) { +// registry.AddFlagBinding(func(fs *pflag.FlagSet) { +// flag := fs.Lookup(flagName) +// if flag == nil { +// // TODO: panic +// +// // TODO: need a way for API consumers to add flags in an +// // OnParse hook, which usually will happen _after_ a call +// // to Configure (viperutil.Value doesn't expose this right +// // now, which you can see the result of in config.go's init +// // func to add the working directory to config-paths). +// } +// +// _ = v.BindPFlag(key, flag) +// +// if flag.Name != key { +// v.RegisterAlias(flag.Name, key) +// } +// }) +// } + +type Static[T any] struct { + *Base[T] +} + +func NewStatic[T any](base *Base[T]) *Static[T] { + base.Bind(registry.Static) + base.get = base.GetFunc(registry.Static) + + return &Static[T]{ + Base: base, + } +} + +func (val *Static[T]) Registry() registry.Bindable { + return registry.Static +} + +type Dynamic[T any] struct { + *Base[T] +} + +func NewDynamic[T any](base *Base[T]) *Dynamic[T] { + base.Bind(registry.Dynamic) + base.get = sync.AdaptGetter(base.Key(), base.GetFunc, registry.Dynamic) + + return &Dynamic[T]{ + Base: base, + } +} + +func (val *Dynamic[T]) Registry() registry.Bindable { + return registry.Dynamic +} diff --git a/go/viperutil/v2/value.go b/go/viperutil/v2/value.go new file mode 100644 index 00000000000..fadd26611de --- /dev/null +++ b/go/viperutil/v2/value.go @@ -0,0 +1,22 @@ +package viperutil + +import ( + "github.com/spf13/pflag" + + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +var ( + _ Value[int] = (*value.Static[int])(nil) + _ Value[int] = (*value.Dynamic[int])(nil) +) + +type Value[T any] interface { + value.Registerable + Get() T + Default() T +} + +func BindFlags(fs *pflag.FlagSet, values ...value.Registerable) { + value.BindFlags(fs, values...) +} diff --git a/go/viperutil/v2/viper.go b/go/viperutil/v2/viper.go new file mode 100644 index 00000000000..6e6d2d3a073 --- /dev/null +++ b/go/viperutil/v2/viper.go @@ -0,0 +1,61 @@ +package viperutil + +import ( + "strings" + + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +type Options[T any] struct { + Aliases []string + FlagName string + EnvVars []string + Default T + + Dynamic bool + + GetFunc func(v *viper.Viper) func(key string) T +} + +func Configure[T any](key string, opts Options[T]) (v Value[T]) { + getfunc := opts.GetFunc + if getfunc == nil { + getfunc = GetFuncForType[T]() + } + + base := &value.Base[T]{ + KeyName: key, + DefaultVal: opts.Default, + GetFunc: getfunc, + Aliases: opts.Aliases, + FlagName: opts.FlagName, + EnvVars: opts.EnvVars, + } + + switch { + case opts.Dynamic: + v = value.NewDynamic(base) + default: + v = value.NewStatic(base) + } + + return v +} + +func KeyPrefixFunc(prefix string) func(subkey string) (fullkey string) { + var keyParts []string + if prefix != "" { + keyParts = append(keyParts, prefix) + } + + return func(subkey string) (fullkey string) { + tmp := keyParts + if subkey != "" { + tmp = append(tmp, subkey) + } + + return strings.Join(tmp, ".") + } +} diff --git a/go/viperutil/v2/viper_test.go b/go/viperutil/v2/viper_test.go new file mode 100644 index 00000000000..4dfdc683eae --- /dev/null +++ b/go/viperutil/v2/viper_test.go @@ -0,0 +1,20 @@ +package viperutil + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestGetFuncForType(t *testing.T) { + v := viper.New() + v.Set("foo", true) + v.Set("bar", 5) + v.Set("baz", []int{1, 2, 3}) + + getBool := GetFuncForType[bool]() + assert.True(t, getBool(v)("foo")) + + assert.Equal(t, 5, GetFuncForType[int]()(v)("bar")) +} diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index 534bc66887a..c2af311ce33 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -44,15 +44,12 @@ import ( "vitess.io/vitess/go/netutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/v2" "vitess.io/vitess/go/vt/grpccommon" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/vterrors" - // register the proper init and shutdown hooks for logging - _ "vitess.io/vitess/go/vt/logutil" - // Include deprecation warnings for soon-to-be-unsupported flag invocations. _flag "vitess.io/vitess/go/internal/flag" ) @@ -329,6 +326,8 @@ func getFlagHooksFor(cmd string) (hooks []func(fs *pflag.FlagSet)) { func ParseFlags(cmd string) { fs := GetFlagSetFor(cmd) + viperutil.BindFlags(fs) + _flag.Parse(fs) if version { @@ -342,7 +341,9 @@ func ParseFlags(cmd string) { log.Exitf("%s doesn't take any positional arguments, got '%s'", cmd, strings.Join(args, " ")) } - readViperConfig(cmd) + if err := viperutil.LoadConfig(); err != nil { + log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) + } logutil.PurgeLogs() } @@ -362,6 +363,8 @@ func GetFlagSetFor(cmd string) *pflag.FlagSet { func ParseFlagsWithArgs(cmd string) []string { fs := GetFlagSetFor(cmd) + viperutil.BindFlags(fs) + _flag.Parse(fs) if version { @@ -374,23 +377,15 @@ func ParseFlagsWithArgs(cmd string) []string { log.Exitf("%s expected at least one positional argument", cmd) } - readViperConfig(cmd) + if err := viperutil.LoadConfig(); err != nil { + log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) + } logutil.PurgeLogs() return args } -func readViperConfig(cmd string) { - if err := viperutil.ReadInDefaultConfig(); err != nil { - log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) - } - - // TODO: MergeConfigMap with servenv's local viper (which does not exist - // yet) before returning. - return -} - // Flag installations for packages that servenv imports. We need to register // here rather than in those packages (which is what we would normally do) // because that would create a dependency cycle. @@ -443,8 +438,8 @@ func init() { OnParse(log.RegisterFlags) // Flags in package logutil are installed for all binaries. OnParse(logutil.RegisterFlags) - // Flags in package viperutil are installed for all binaries. - OnParse(viperutil.RegisterDefaultConfigFlags) + // Flags in package viperutil/config are installed for all binaries. + OnParse(viperutil.RegisterFlags) } func RegisterFlagsForTopoBinaries(registerFlags func(fs *pflag.FlagSet)) { From 26cc5510b82da9d9bcb78220404fead85fdfba1b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Nov 2022 15:48:54 -0500 Subject: [PATCH 021/100] WIP WIP WIP: update trace pkg to use new api (mostly) Signed-off-by: Andrew Mason --- go/trace/trace.go | 28 ++++++++++++++-------------- go/trace/trace_test.go | 24 +++++++++++------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/go/trace/trace.go b/go/trace/trace.go index 6892da815fc..d2d1d126d96 100644 --- a/go/trace/trace.go +++ b/go/trace/trace.go @@ -26,10 +26,9 @@ import ( "strings" "github.com/spf13/pflag" - "github.com/spf13/viper" "google.golang.org/grpc" - "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/v2" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" ) @@ -137,29 +136,30 @@ var ( /* flags */ - configKey = viperutil.KeyPartial(configKeyPrefix) + configKey = viperutil.KeyPrefixFunc(configKeyPrefix) - tracingServer = viperutil.NewValue( + tracingServer = viperutil.Configure( configKey("service"), - viper.GetString, - viperutil.WithFlags[string]("tracer"), - viperutil.WithDefault("noop"), + viperutil.Options[string]{ + Default: "noop", + FlagName: "tracer", + }, ) - enableLogging = viperutil.NewValue( + enableLogging = viperutil.Configure( configKey("enable-logging"), - viper.GetBool, - viperutil.WithFlags[bool]("tracing-enable-logging"), + viperutil.Options[bool]{ + FlagName: "tracing-enable-logging", + }, ) pluginFlags []func(fs *pflag.FlagSet) ) func RegisterFlags(fs *pflag.FlagSet) { - fs.String("tracer", tracingServer.Value(), "tracing service to use") - tracingServer.Bind(nil, fs) - + fs.String("tracer", tracingServer.Default(), "tracing service to use") fs.Bool("tracing-enable-logging", false, "whether to enable logging in the tracing service") - enableLogging.Bind(nil, fs) + + viperutil.BindFlags(fs, tracingServer, enableLogging) for _, fn := range pluginFlags { fn(fs) diff --git a/go/trace/trace_test.go b/go/trace/trace_test.go index 7649c4f65c0..eb4ccf11aa0 100644 --- a/go/trace/trace_test.go +++ b/go/trace/trace_test.go @@ -23,10 +23,7 @@ import ( "strings" "testing" - "github.com/spf13/viper" "google.golang.org/grpc" - - "vitess.io/vitess/go/viperutil" ) func TestFakeSpan(t *testing.T) { @@ -52,16 +49,17 @@ func TestRegisterService(t *testing.T) { return tracer, tracer, nil } - var old viperutil.Value[string] - old = *tracingServer - t.Cleanup(func() { - *tracingServer = old - }) - - v := viper.New() - tracingServer = viperutil.NewValue(configKey("service"), v.GetString) - v.Set(configKey("service"), fakeName) - tracingServer.Fetch() + // TODO: need a test-stub API in viperutil/v2 + // var old viperutil.Value[string] + // old = *tracingServer + // t.Cleanup(func() { + // *tracingServer = old + // }) + // + // v := viper.New() + // tracingServer = viperutil.NewValue(configKey("service"), v.GetString) + // v.Set(configKey("service"), fakeName) + // tracingServer.Fetch() serviceName := "vtservice" closer := StartTracing(serviceName) From 4d3fdca8e52e23ab2ba18932346c55c4e7ce5759 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Nov 2022 15:49:30 -0500 Subject: [PATCH 022/100] WIP: example of dynamic variable and getting notified of config changes Signed-off-by: Andrew Mason --- go/vt/topo/locks.go | 73 ++++++++++++++++----------------------------- 1 file changed, 25 insertions(+), 48 deletions(-) diff --git a/go/vt/topo/locks.go b/go/vt/topo/locks.go index 7073b970fe0..7e935ab7792 100644 --- a/go/vt/topo/locks.go +++ b/go/vt/topo/locks.go @@ -26,12 +26,10 @@ import ( "time" "github.com/spf13/pflag" - "github.com/spf13/viper" _flag "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/viperutil" - "vitess.io/vitess/go/viperutil/vipersync" + "vitess.io/vitess/go/viperutil/v2" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/servenv" @@ -46,19 +44,13 @@ var ( // shard / keyspace lock can be acquired for. LockTimeout = 45 * time.Second - sv = vipersync.New() - __getRemoteOperationTimeout = vipersync.AdaptGetter(sv, "topo.remote_operation_timeout", func(v *viper.Viper) func(key string) time.Duration { - return v.GetDuration - }) - remoteOperationTimeout = viperutil.NewValue( + remoteOperationTimeout = viperutil.Configure( "topo.remote_operation_timeout", - - __getRemoteOperationTimeout, - // vipersync.AdaptGetter(sv, "topo.remote_operation_timeout", func(v *viper.Viper) func(key string) time.Duration { - // return v.GetDuration - // }), - viperutil.WithDefault(30*time.Second), - viperutil.WithFlags[time.Duration]("remote_operation_timeout"), + viperutil.Options[time.Duration]{ + Default: 30 * time.Second, + FlagName: "remote_operation_timeout", + Dynamic: true, + }, ) // RemoteOperationTimeout is used for operations where we have to @@ -84,46 +76,31 @@ func init() { for _, cmd := range FlagBinaries { servenv.OnParseFor(cmd, registerTopoLockFlags) } -} - -func registerTopoLockFlags(fs *pflag.FlagSet) { - fs.DurationVar(&RemoteOperationTimeout, "remote_operation_timeout", RemoteOperationTimeout, "time to wait for a remote operation") - fs.DurationVar(&LockTimeout, "lock-timeout", LockTimeout, "Maximum time for which a shard/keyspace lock can be acquired for") - - remoteOperationTimeout.Bind(nil, fs) - servenv.OnRun(func() { - log.Errorf("servenv.OnRun hook for topo called") - vipersync.BindValue(sv, remoteOperationTimeout, fs) - - cfg := viper.ConfigFileUsed() - if cfg == "" { - log.Errorf("no config file set on global viper") - return + ch := make(chan struct{}, 1) + go func() { + for range ch { + log.Errorf("topo.RemoteOperationTimeout updated; new value: %v", remoteOperationTimeout.Get()) } - ch := make(chan struct{}, 1) - go func() { - for range ch { - log.Errorf("topo.RemoteOperationTimeout updated; new value: %v", remoteOperationTimeout.Fetch()) - } + log.Errorf("topo settings channel closed") + }() - log.Errorf("topo settings channel closed") - }() + servenv.OnTerm(func() { + close(ch) + }) - servenv.OnTerm(func() { - close(ch) - }) + viperutil.NotifyConfigReload(ch) - sv.Notify(ch) - if err := sv.Watch(viper.ConfigFileUsed()); err != nil { - log.Errorf("failed to read config in vipersync.disk: %s", err.Error()) - return - } + log.Errorf("initialized config watcher for topo settings") + log.Errorf("initial topo.RemoteOperationTimeout: %v", remoteOperationTimeout.Get()) +} - log.Errorf("initialized config watcher for topo settings") - log.Errorf("initial topo.RemoteOperationTimeout: %v", remoteOperationTimeout.Fetch()) - }) +func registerTopoLockFlags(fs *pflag.FlagSet) { + fs.DurationVar(&RemoteOperationTimeout, "remote_operation_timeout", RemoteOperationTimeout, "time to wait for a remote operation") + viperutil.BindFlags(fs, remoteOperationTimeout) + + fs.DurationVar(&LockTimeout, "lock-timeout", LockTimeout, "Maximum time for which a shard/keyspace lock can be acquired for") } // newLock creates a new Lock. From bada60a18bdaf1e42a6ddc7446ca9f8f864ea49e Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Nov 2022 18:57:17 -0500 Subject: [PATCH 023/100] fix default workdir override Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go index 4dbf6f2ceed..292714dcb08 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/v2/config.go @@ -64,6 +64,8 @@ func init() { } configPaths.(*value.Static[[]string]).DefaultVal = []string{wd} + // Need to re-trigger the SetDefault call done during Configure. + registry.Static.SetDefault(configPaths.Key(), configPaths.Default()) } func RegisterFlags(fs *pflag.FlagSet) { From d8ee263d36add8c9e0415b5bdc4ccaa6f073cce0 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 8 Nov 2022 18:57:42 -0500 Subject: [PATCH 024/100] fix watch for when no config file was used Signed-off-by: Andrew Mason --- go/viperutil/v2/internal/sync/sync.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/viperutil/v2/internal/sync/sync.go b/go/viperutil/v2/internal/sync/sync.go index 6409f43bf98..ac9c1b4e2f7 100644 --- a/go/viperutil/v2/internal/sync/sync.go +++ b/go/viperutil/v2/internal/sync/sync.go @@ -35,7 +35,13 @@ func (v *Viper) Watch(static *viper.Viper) error { return errors.New("duplicate watch") } - v.disk.SetConfigFile(static.ConfigFileUsed()) + cfg := static.ConfigFileUsed() + if cfg == "" { + // No config file to watch, just merge the settings and return. + return v.live.MergeConfigMap(static.AllSettings()) + } + + v.disk.SetConfigFile(cfg) if err := v.disk.ReadInConfig(); err != nil { return err } From ab160d1b31b03e02d2f07a2f277b4872882d916a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 9 Nov 2022 12:14:18 -0500 Subject: [PATCH 025/100] provide mechanism to stub values for tests Signed-off-by: Andrew Mason --- go/viperutil/v2/internal/value/value.go | 10 +++--- go/viperutil/v2/vipertest/stub.go | 42 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 go/viperutil/v2/vipertest/stub.go diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/v2/internal/value/value.go index 2b60ed63b0f..925141fe552 100644 --- a/go/viperutil/v2/internal/value/value.go +++ b/go/viperutil/v2/internal/value/value.go @@ -21,8 +21,8 @@ type Base[T any] struct { KeyName string DefaultVal T - GetFunc func(v *viper.Viper) func(key string) T - get func(key string) T + GetFunc func(v *viper.Viper) func(key string) T + BoundGetFunc func(key string) T Aliases []string FlagName string @@ -31,7 +31,7 @@ type Base[T any] struct { func (val *Base[T]) Key() string { return val.KeyName } func (val *Base[T]) Default() T { return val.DefaultVal } -func (val *Base[T]) Get() T { return val.get(val.Key()) } +func (val *Base[T]) Get() T { return val.BoundGetFunc(val.Key()) } var errNoFlagDefined = errors.New("flag not defined") @@ -106,7 +106,7 @@ type Static[T any] struct { func NewStatic[T any](base *Base[T]) *Static[T] { base.Bind(registry.Static) - base.get = base.GetFunc(registry.Static) + base.BoundGetFunc = base.GetFunc(registry.Static) return &Static[T]{ Base: base, @@ -123,7 +123,7 @@ type Dynamic[T any] struct { func NewDynamic[T any](base *Base[T]) *Dynamic[T] { base.Bind(registry.Dynamic) - base.get = sync.AdaptGetter(base.Key(), base.GetFunc, registry.Dynamic) + base.BoundGetFunc = sync.AdaptGetter(base.Key(), base.GetFunc, registry.Dynamic) return &Dynamic[T]{ Base: base, diff --git a/go/viperutil/v2/vipertest/stub.go b/go/viperutil/v2/vipertest/stub.go new file mode 100644 index 00000000000..5d6c37d9771 --- /dev/null +++ b/go/viperutil/v2/vipertest/stub.go @@ -0,0 +1,42 @@ +package vipertest + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +func Stub[T any](t *testing.T, v *viper.Viper, val viperutil.Value[T]) (reset func()) { + t.Helper() + + reset = func() {} + if !assert.False(t, v.InConfig(val.Key()), "value for key %s already stubbed", val.Key()) { + return func() {} + } + + var base *value.Base[T] + switch val := val.(type) { + case *value.Static[T]: + base = val.Base + case *value.Dynamic[T]: + base = val.Base + default: + assert.Fail(t, "value %+v does not support stubbing", val) + return func() {} + } + + oldGet := base.BoundGetFunc + base.BoundGetFunc = base.GetFunc(v) + + return func() { + base.BoundGetFunc = oldGet + } +} + +// func StubValues(v *viper.Viper, values ...any) (reset func()) { +// +// } From 9d7f09259e53fd84913e03302099299474c20a61 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 9 Nov 2022 12:30:00 -0500 Subject: [PATCH 026/100] update usage in go/trace to new api Signed-off-by: Andrew Mason --- go/trace/plugin_datadog.go | 24 ++++++++++---------- go/trace/plugin_jaeger.go | 45 +++++++++++++++++++------------------- go/trace/trace_test.go | 20 +++++++---------- 3 files changed, 42 insertions(+), 47 deletions(-) diff --git a/go/trace/plugin_datadog.go b/go/trace/plugin_datadog.go index 53ab85ae8b2..7480b9d85a9 100644 --- a/go/trace/plugin_datadog.go +++ b/go/trace/plugin_datadog.go @@ -6,25 +6,26 @@ import ( "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" - "github.com/spf13/viper" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer" ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/v2" ) var ( - dataDogConfigKey = viperutil.KeyPartial(configKey("datadog")) + dataDogConfigKey = viperutil.KeyPrefixFunc(configKey("datadog")) - dataDogHost = viperutil.NewValue( + dataDogHost = viperutil.Configure( dataDogConfigKey("agent.host"), - viper.GetString, - viperutil.WithFlags[string]("datadog-agent-host"), + viperutil.Options[string]{ + FlagName: "datadog-agent-host", + }, ) - dataDogPort = viperutil.NewValue( + dataDogPort = viperutil.Configure( dataDogConfigKey("agent.port"), - viper.GetString, - viperutil.WithFlags[string]("datadog-agent-port"), + viperutil.Options[string]{ + FlagName: "datadog-agent-port", + }, ) ) @@ -33,10 +34,9 @@ func init() { // includes datadaog tracing flags. pluginFlags = append(pluginFlags, func(fs *pflag.FlagSet) { fs.String("datadog-agent-host", "", "host to send spans to. if empty, no tracing will be done") - dataDogHost.Bind(nil, fs) - fs.String("datadog-agent-port", "", "port to send spans to. if empty, no tracing will be done") - dataDogPort.Bind(nil, fs) + + viperutil.BindFlags(fs, dataDogHost, dataDogPort) }) } diff --git a/go/trace/plugin_jaeger.go b/go/trace/plugin_jaeger.go index a4edd247378..1341d637481 100644 --- a/go/trace/plugin_jaeger.go +++ b/go/trace/plugin_jaeger.go @@ -21,10 +21,9 @@ import ( "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" - "github.com/spf13/viper" "github.com/uber/jaeger-client-go/config" - "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/v2" "vitess.io/vitess/go/vt/log" ) @@ -35,25 +34,28 @@ included but nothing Jaeger specific. */ var ( - jaegerConfigKey = viperutil.KeyPartial(configKey("jaeger")) - agentHost = viperutil.NewValue( + jaegerConfigKey = viperutil.KeyPrefixFunc(configKey("jaeger")) + agentHost = viperutil.Configure( jaegerConfigKey("agent-host"), - viper.GetString, - viperutil.WithFlags[string]("jaeger-agent-host"), + viperutil.Options[string]{ + FlagName: "jaeger-agent-host", + }, ) - samplingType = viperutil.NewValue( + samplingType = viperutil.Configure( jaegerConfigKey("sampling_type"), - viper.GetString, - viperutil.WithFlags[string]("tracing-sampling-type"), - viperutil.WithDefault("const"), - viperutil.WithEnvVars[string]("JAEGER_SAMPLER_TYPE"), + viperutil.Options[string]{ + Default: "const", + EnvVars: []string{"JAEGER_SAMPLER_TYPE"}, + FlagName: "tracing-sampling-type", + }, ) - samplingRate = viperutil.NewValue( + samplingRate = viperutil.Configure( jaegerConfigKey("sampling_rate"), - viper.GetFloat64, - viperutil.WithFlags[float64]("tracing-sampling-rate"), - viperutil.WithDefault(0.1), - viperutil.WithEnvVars[float64]("JAEGER_SAMPLER_PARAM"), + viperutil.Options[float64]{ + Default: 0.1, + EnvVars: []string{"JAEGER_SAMPLER_PARAM"}, + FlagName: "tracing-sampling-rate", + }, ) ) @@ -61,14 +63,11 @@ func init() { // If compiled with plugin_jaeger, ensure that trace.RegisterFlags includes // jaeger tracing flags. pluginFlags = append(pluginFlags, func(fs *pflag.FlagSet) { - fs.String("jaeger-agent-host", "", "host and port to send spans to. if empty, no tracing will be done") - agentHost.Bind(nil, fs) + fs.String("jaeger-agent-host", agentHost.Default(), "host and port to send spans to. if empty, no tracing will be done") + fs.String("tracing-sampling-type", samplingType.Default(), "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") + fs.Float64("tracing-sampling-rate", samplingRate.Default(), "sampling rate for the probabilistic jaeger sampler") - fs.String("tracing-sampling-type", samplingType.Value(), "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") - samplingType.Bind(nil, fs) - - fs.Float64("tracing-sampling-rate", samplingRate.Value(), "sampling rate for the probabilistic jaeger sampler") - samplingRate.Bind(nil, fs) + viperutil.BindFlags(fs, agentHost, samplingRate, samplingType) }) } diff --git a/go/trace/trace_test.go b/go/trace/trace_test.go index eb4ccf11aa0..5605af18d3f 100644 --- a/go/trace/trace_test.go +++ b/go/trace/trace_test.go @@ -23,7 +23,10 @@ import ( "strings" "testing" + "github.com/spf13/viper" "google.golang.org/grpc" + + "vitess.io/vitess/go/viperutil/v2/vipertest" ) func TestFakeSpan(t *testing.T) { @@ -49,23 +52,16 @@ func TestRegisterService(t *testing.T) { return tracer, tracer, nil } - // TODO: need a test-stub API in viperutil/v2 - // var old viperutil.Value[string] - // old = *tracingServer - // t.Cleanup(func() { - // *tracingServer = old - // }) - // - // v := viper.New() - // tracingServer = viperutil.NewValue(configKey("service"), v.GetString) - // v.Set(configKey("service"), fakeName) - // tracingServer.Fetch() + v := viper.New() + t.Cleanup(vipertest.Stub(t, v, tracingServer)) + + v.Set(tracingServer.Key(), fakeName) serviceName := "vtservice" closer := StartTracing(serviceName) tracer, ok := closer.(*fakeTracer) if !ok { - t.Fatalf("did not get the expected tracer") + t.Fatalf("did not get the expected tracer, got %+v (%T)", tracer, tracer) } if tracer.name != serviceName { From 9710c891b5d411858b165562ddf6da4c41ad6038 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 10 Nov 2022 06:21:47 -0500 Subject: [PATCH 027/100] update azblobbackupstorage to use new API Signed-off-by: Andrew Mason --- go/vt/mysqlctl/azblobbackupstorage/azblob.go | 62 ++++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 10134c5b73d..0b1c2eb33fe 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -31,9 +31,8 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/spf13/pflag" - "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/v2" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" @@ -42,60 +41,59 @@ import ( var ( // This is the account name - accountName = viperutil.NewValue( + accountName = viperutil.Configure( configKey("account.name"), - viper.GetString, - viperutil.WithFlags[string]("azblob_backup_account_name"), - viperutil.WithEnvVars[string]("VT_AZBLOB_ACCOUNT_NAME"), + viperutil.Options[string]{ + EnvVars: []string{"VT_AZBLOB_ACCOUNT_NAME"}, + FlagName: "azblob_backup_account_name", + }, ) // This is the private access key - accountKeyFile = viperutil.NewValue( + accountKeyFile = viperutil.Configure( configKey("account.key_file"), - viper.GetString, - viperutil.WithFlags[string]("azblob_backup_account_key_file"), + viperutil.Options[string]{ + FlagName: "azblob_backup_account_key_file", + }, ) // This is the name of the container that will store the backups - containerName = viperutil.NewValue( + containerName = viperutil.Configure( configKey("container_name"), - viper.GetString, - viperutil.WithFlags[string]("azblob_backup_container_name"), + viperutil.Options[string]{ + FlagName: "azblob_backup_container_name", + }, ) // This is an optional prefix to prepend to all files - storageRoot = viperutil.NewValue( + storageRoot = viperutil.Configure( configKey("storage_root"), - viper.GetString, - viperutil.WithFlags[string]("azblob_backup_storage_root"), + viperutil.Options[string]{ + FlagName: "azblob_backup_storage_root", + }, ) - azBlobParallelism = viperutil.NewValue( + azBlobParallelism = viperutil.Configure( configKey("parallelism"), - viper.GetInt, - viperutil.WithFlags[int]("azblob_backup_parallelism"), + viperutil.Options[int]{ + Default: 1, + FlagName: "azblob_backup_parallelism", + }, ) ) const configKeyPrefix = "backup.storage.azblob" -var configKey = viperutil.KeyPartial(configKeyPrefix) +var configKey = viperutil.KeyPrefixFunc(configKeyPrefix) func registerFlags(fs *pflag.FlagSet) { - fs.String("azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") - accountName.Bind(nil, fs) - - fs.String("azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") - accountKeyFile.Bind(nil, fs) - - fs.String("azblob_backup_container_name", "", "Azure Blob Container Name.") - containerName.Bind(nil, fs) - - fs.String("azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") - storageRoot.Bind(nil, fs) + fs.String("azblob_backup_account_name", accountName.Default(), "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") + fs.String("azblob_backup_account_key_file", accountKeyFile.Default(), "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") + fs.String("azblob_backup_container_name", containerName.Default(), "Azure Blob Container Name.") + fs.String("azblob_backup_storage_root", storageRoot.Default(), "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") + fs.Int("azblob_backup_parallelism", azBlobParallelism.Default(), "Azure Blob operation parallelism (requires extra memory when increased).") - fs.Int("azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased).") - azBlobParallelism.Bind(nil, fs) + viperutil.BindFlags(fs, accountName, accountKeyFile, containerName, storageRoot, azBlobParallelism) } func init() { From 0f4a6c6c6926b362b96633df087bc8702bb1b3a5 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 10 Nov 2022 10:12:06 -0500 Subject: [PATCH 028/100] fix config-handling parsing Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 2 +- go/viperutil/v2/config_test.go | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go index 292714dcb08..1c15f05ff26 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/v2/config.go @@ -162,7 +162,7 @@ func decodeHandlingValue(from, to reflect.Type, data any) (any, error) { } switch { - case from.ConvertibleTo(reflect.TypeOf(h)): + case from == reflect.TypeOf(h): return data.(ConfigFileNotFoundHandling), nil case from.Kind() == reflect.Int: return ConfigFileNotFoundHandling(data.(int)), nil diff --git a/go/viperutil/v2/config_test.go b/go/viperutil/v2/config_test.go index 151a9f05b3e..19a3b3c6754 100644 --- a/go/viperutil/v2/config_test.go +++ b/go/viperutil/v2/config_test.go @@ -11,6 +11,7 @@ import ( func TestGetConfigHandlingValue(t *testing.T) { v := viper.New() + v.SetDefault("default", ExitOnConfigFileNotFound) v.SetConfigType("yaml") cfg := ` @@ -27,5 +28,6 @@ duration: 10h assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("bar"), "failed to get int-like string value") assert.Equal(t, ErrorOnConfigFileNotFound, getHandlingValueFunc("baz"), "failed to get string value") assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("notset"), "failed to get value on unset key") - assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("duration"), "failed to get value on duration value") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("duration"), "failed to get value on duration key") + assert.Equal(t, ExitOnConfigFileNotFound, getHandlingValueFunc("default"), "failed to get value on default key") } From 1cb6ef5ec14af1d9c2c9ad4af5e98038eca161fb Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 10 Nov 2022 10:12:28 -0500 Subject: [PATCH 029/100] add test for config watching Signed-off-by: Andrew Mason --- go/viperutil/v2/internal/sync/sync_test.go | 137 +++++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 go/viperutil/v2/internal/sync/sync_test.go diff --git a/go/viperutil/v2/internal/sync/sync_test.go b/go/viperutil/v2/internal/sync/sync_test.go new file mode 100644 index 00000000000..9382e5709c4 --- /dev/null +++ b/go/viperutil/v2/internal/sync/sync_test.go @@ -0,0 +1,137 @@ +package sync_test + +import ( + "context" + "encoding/json" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/viperutil/v2" + vipersync "vitess.io/vitess/go/viperutil/v2/internal/sync" + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +func TestWatchConfig(t *testing.T) { + type config struct { + A, B int + } + + tmp, err := os.CreateTemp(".", "TestWatchConfig_*.json") + require.NoError(t, err) + t.Cleanup(func() { os.Remove(tmp.Name()) }) + + stat, err := os.Stat(tmp.Name()) + require.NoError(t, err) + + writeConfig := func(a, b int) error { + data, err := json.Marshal(&config{A: a, B: b}) + if err != nil { + return err + } + + return os.WriteFile(tmp.Name(), data, stat.Mode()) + } + writeRandomConfig := func() error { + a, b := rand.Intn(100), rand.Intn(100) + return writeConfig(a, b) + } + + require.NoError(t, writeRandomConfig()) + + v := viper.New() + v.SetConfigFile(tmp.Name()) + require.NoError(t, v.ReadInConfig()) + + wCh, rCh := make(chan struct{}), make(chan struct{}) + v.OnConfigChange(func(in fsnotify.Event) { + select { + case <-rCh: + return + default: + } + + wCh <- struct{}{} + // block forever to prevent this viper instance from double-updating. + <-rCh + }) + v.WatchConfig() + + // Make sure that basic, unsynchronized WatchConfig is set up before + // beginning the actual test. + a, b := v.GetInt("a"), v.GetInt("b") + require.NoError(t, writeConfig(a+1, b+1)) + <-wCh // wait for the update to finish + + require.Equal(t, a+1, v.GetInt("a")) + require.Equal(t, b+1, v.GetInt("b")) + + rCh <- struct{}{} + + sv := vipersync.New() + A := viperutil.Configure("a", viperutil.Options[int]{Dynamic: true}) + B := viperutil.Configure("b", viperutil.Options[int]{Dynamic: true}) + + A.(*value.Dynamic[int]).Base.BoundGetFunc = vipersync.AdaptGetter("a", func(v *viper.Viper) func(key string) int { + return v.GetInt + }, sv) + B.(*value.Dynamic[int]).Base.BoundGetFunc = vipersync.AdaptGetter("b", func(v *viper.Viper) func(key string) int { + return v.GetInt + }, sv) + + require.NoError(t, sv.Watch(v)) + + var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + + // Sleep between 25 and 50ms between reads. + readJitter := func() time.Duration { + return time.Duration(jitter(25, 50)) * time.Millisecond + } + // Sleep between 75 and 125ms between writes. + writeJitter := func() time.Duration { + return time.Duration(jitter(75, 125)) * time.Millisecond + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + default: + } + + switch i % 2 { + case 0: + A.Get() + case 1: + B.Get() + } + + time.Sleep(readJitter()) + } + }(i) + } + + for i := 0; i < 100; i++ { + require.NoError(t, writeRandomConfig()) + time.Sleep(writeJitter()) + } + + cancel() + wg.Wait() +} + +func jitter(min, max int) int { + return min + rand.Intn(max-min+1) +} From 99dba7f7c1b8d4c870e245a3fb88ad3721813801 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 10 Nov 2022 14:35:16 -0500 Subject: [PATCH 030/100] cleaning up some TODO markers Signed-off-by: Andrew Mason --- go/viperutil/v2/errors.go | 11 +++++++++++ go/viperutil/v2/internal/sync/sync.go | 6 +++--- go/viperutil/v2/internal/value/value.go | 26 ++----------------------- 3 files changed, 16 insertions(+), 27 deletions(-) create mode 100644 go/viperutil/v2/errors.go diff --git a/go/viperutil/v2/errors.go b/go/viperutil/v2/errors.go new file mode 100644 index 00000000000..f1ebd91e41f --- /dev/null +++ b/go/viperutil/v2/errors.go @@ -0,0 +1,11 @@ +package viperutil + +import ( + "vitess.io/vitess/go/viperutil/v2/internal/sync" + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +var ( + ErrDuplicateWatch = sync.ErrDuplicateWatch + ErrNoFlagDefined = value.ErrNoFlagDefined +) diff --git a/go/viperutil/v2/internal/sync/sync.go b/go/viperutil/v2/internal/sync/sync.go index ac9c1b4e2f7..4c064f2cc84 100644 --- a/go/viperutil/v2/internal/sync/sync.go +++ b/go/viperutil/v2/internal/sync/sync.go @@ -28,11 +28,11 @@ func New() *Viper { } } +var ErrDuplicateWatch = errors.New("duplicate watch") + func (v *Viper) Watch(static *viper.Viper) error { if v.watchingConfig { - // TODO: declare an exported error and include the filename in wrapped - // error. - return errors.New("duplicate watch") + return fmt.Errorf("%w: viper is already watching %s", ErrDuplicateWatch, v.disk.ConfigFileUsed()) } cfg := static.ConfigFileUsed() diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/v2/internal/value/value.go index 925141fe552..45a04507240 100644 --- a/go/viperutil/v2/internal/value/value.go +++ b/go/viperutil/v2/internal/value/value.go @@ -33,7 +33,7 @@ func (val *Base[T]) Key() string { return val.KeyName } func (val *Base[T]) Default() T { return val.DefaultVal } func (val *Base[T]) Get() T { return val.BoundGetFunc(val.Key()) } -var errNoFlagDefined = errors.New("flag not defined") +var ErrNoFlagDefined = errors.New("flag not defined") func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { if val.FlagName == "" { @@ -42,7 +42,7 @@ func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { flag := fs.Lookup(val.FlagName) if flag == nil { - return nil, fmt.Errorf("%w with name %s (for key %s)", errNoFlagDefined, val.FlagName, val.Key()) // TODO: export error + return nil, fmt.Errorf("%w with name %s (for key %s)", ErrNoFlagDefined, val.FlagName, val.Key()) } return flag, nil @@ -66,7 +66,6 @@ func BindFlags(fs *pflag.FlagSet, values ...Registerable) { flag, err := val.Flag(fs) switch { case err != nil: - // TODO: panic panic(fmt.Errorf("failed to load flag for %s: %w", val.Key(), err)) case flag == nil: continue @@ -79,27 +78,6 @@ func BindFlags(fs *pflag.FlagSet, values ...Registerable) { } } -// func xxx_DELETE_ME_bindFlag(v registry.Bindable, key string, flagName string) { -// registry.AddFlagBinding(func(fs *pflag.FlagSet) { -// flag := fs.Lookup(flagName) -// if flag == nil { -// // TODO: panic -// -// // TODO: need a way for API consumers to add flags in an -// // OnParse hook, which usually will happen _after_ a call -// // to Configure (viperutil.Value doesn't expose this right -// // now, which you can see the result of in config.go's init -// // func to add the working directory to config-paths). -// } -// -// _ = v.BindPFlag(key, flag) -// -// if flag.Name != key { -// v.RegisterAlias(flag.Name, key) -// } -// }) -// } - type Static[T any] struct { *Base[T] } From 57366b0b53c7e472f14c309e44eb2f58c513f199 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Thu, 10 Nov 2022 15:31:41 -0500 Subject: [PATCH 031/100] starting to work on doc comments Signed-off-by: Andrew Mason --- go/viperutil/v2/internal/registry/registry.go | 12 +++++++++++- go/viperutil/v2/internal/value/value.go | 10 ++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/go/viperutil/v2/internal/registry/registry.go b/go/viperutil/v2/internal/registry/registry.go index 0efc7e3ce6c..fdac6e857a1 100644 --- a/go/viperutil/v2/internal/registry/registry.go +++ b/go/viperutil/v2/internal/registry/registry.go @@ -8,13 +8,23 @@ import ( ) var ( - Static = viper.New() + // Static is the registry for static config variables. These variables will + // never be affected by a Watch-ed config, and maintain their original + // values for the lifetime of the process. + Static = viper.New() + // Dynamic is the registry for dynamic config variables. If a config file is + // found by viper, it will be watched by a threadsafe wrapper around a + // second viper (see sync.Viper), and variables registered to it will pick + // up changes to that config file throughout the lifetime of the process. Dynamic = sync.New() _ Bindable = (*viper.Viper)(nil) _ Bindable = (*sync.Viper)(nil) ) +// Bindable represents the methods needed to bind a value.Value to a given +// registry. It exists primarly to allow us to treat a sync.Viper as a +// viper.Viper for configuration registration purposes. type Bindable interface { BindEnv(vars ...string) error BindPFlag(key string, flag *pflag.Flag) error diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/v2/internal/value/value.go index 45a04507240..00273f129f0 100644 --- a/go/viperutil/v2/internal/value/value.go +++ b/go/viperutil/v2/internal/value/value.go @@ -11,12 +11,20 @@ import ( "vitess.io/vitess/go/viperutil/v2/internal/sync" ) +// Registerable is the subset of the interface exposed by Values (which is +// declared in the public viperutil package). +// +// We need a separate interface type because Go generics do not let you define +// a function that takes Value[T] for many, different T's, which we want to do +// for BindFlags. type Registerable interface { Key() string Registry() registry.Bindable Flag(fs *pflag.FlagSet) (*pflag.Flag, error) } +// Base is the base functionality shared by Static and Dynamic values. It +// implements viperutil.Value. type Base[T any] struct { KeyName string DefaultVal T @@ -33,6 +41,8 @@ func (val *Base[T]) Key() string { return val.KeyName } func (val *Base[T]) Default() T { return val.DefaultVal } func (val *Base[T]) Get() T { return val.BoundGetFunc(val.Key()) } +// ErrNoFlagDefined is returned when a Value has a FlagName set, but the given +// FlagSet does not define a flag with that name. var ErrNoFlagDefined = errors.New("flag not defined") func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { From 6722cf2889478eb0a3563259a6614f6458979473 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 11 Nov 2022 19:59:35 -0500 Subject: [PATCH 032/100] more docs Signed-off-by: Andrew Mason --- go/viperutil/v2/internal/value/value.go | 28 ++++++++++++++++++++++--- go/viperutil/v2/vipertest/stub.go | 11 ++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/v2/internal/value/value.go index 00273f129f0..36c441d044a 100644 --- a/go/viperutil/v2/internal/value/value.go +++ b/go/viperutil/v2/internal/value/value.go @@ -45,6 +45,13 @@ func (val *Base[T]) Get() T { return val.BoundGetFunc(val.Key()) } // FlagSet does not define a flag with that name. var ErrNoFlagDefined = errors.New("flag not defined") +// Flag is part of the Registerable interface. If the given flag set has a flag +// with the name of this value's configured flag, that flag is returned, along +// with a nil error. If no flag exists on the flag set with that name, an error +// is returned. +// +// If the value is not configured to correspond to a flag (FlagName == ""), then +// (nil, nil) is returned. func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { if val.FlagName == "" { return nil, nil @@ -58,7 +65,7 @@ func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { return flag, nil } -func (val *Base[T]) Bind(v registry.Bindable) { +func (val *Base[T]) bind(v registry.Bindable) { v.SetDefault(val.Key(), val.DefaultVal) for _, alias := range val.Aliases { @@ -71,6 +78,9 @@ func (val *Base[T]) Bind(v registry.Bindable) { } } +// BindFlags creates bindings between each value's registry and the given flag +// set. This function will panic if any of the values defines a flag that does +// not exist in the flag set. func BindFlags(fs *pflag.FlagSet, values ...Registerable) { for _, val := range values { flag, err := val.Flag(fs) @@ -88,12 +98,17 @@ func BindFlags(fs *pflag.FlagSet, values ...Registerable) { } } +// Static is a static value. Static values register to the Static registry, and +// do not respond to changes to config files. Their Get() method will return the +// same value for the lifetime of the process. type Static[T any] struct { *Base[T] } +// NewStatic returns a static value derived from the given base value, after +// binding it to the static registry. func NewStatic[T any](base *Base[T]) *Static[T] { - base.Bind(registry.Static) + base.bind(registry.Static) base.BoundGetFunc = base.GetFunc(registry.Static) return &Static[T]{ @@ -105,12 +120,19 @@ func (val *Static[T]) Registry() registry.Bindable { return registry.Static } +// Dynamic is a dynamic value. Dynamic values register to the Dynamic registry, +// and respond to changes to watched config files. Their Get() methods will +// return whatever value is currently live in the config, in a threadsafe +// manner. type Dynamic[T any] struct { *Base[T] } +// NewDynamic returns a dynamic value derived from the given base value, after +// binding it to the dynamic registry and wrapping its GetFunc to be threadsafe +// with respect to config reloading. func NewDynamic[T any](base *Base[T]) *Dynamic[T] { - base.Bind(registry.Dynamic) + base.bind(registry.Dynamic) base.BoundGetFunc = sync.AdaptGetter(base.Key(), base.GetFunc, registry.Dynamic) return &Dynamic[T]{ diff --git a/go/viperutil/v2/vipertest/stub.go b/go/viperutil/v2/vipertest/stub.go index 5d6c37d9771..51921443b29 100644 --- a/go/viperutil/v2/vipertest/stub.go +++ b/go/viperutil/v2/vipertest/stub.go @@ -10,6 +10,13 @@ import ( "vitess.io/vitess/go/viperutil/v2/internal/value" ) +// Stub stubs out a given value to use the passed-in viper to retrieve its +// config value for testing purposes. It returns a function to undo this, +// resetting the Value to whatever registry (Static, or Dynamic) it was +// originally bound to. +// +// It fails the test if a caller attempts to stub the same value multiple times +// to a particular viper. func Stub[T any](t *testing.T, v *viper.Viper, val viperutil.Value[T]) (reset func()) { t.Helper() @@ -36,7 +43,3 @@ func Stub[T any](t *testing.T, v *viper.Viper, val viperutil.Value[T]) (reset fu base.BoundGetFunc = oldGet } } - -// func StubValues(v *viper.Viper, values ...any) (reset func()) { -// -// } From 02a976dbe3ac030ae2549ed080a0b36980017266 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 12 Nov 2022 06:39:03 -0500 Subject: [PATCH 033/100] document GetPath and add example test Signed-off-by: Andrew Mason --- go/viperutil/v2/funcs/get.go | 2 ++ go/viperutil/v2/funcs/get_test.go | 33 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 go/viperutil/v2/funcs/get_test.go diff --git a/go/viperutil/v2/funcs/get.go b/go/viperutil/v2/funcs/get.go index 464a5ccad73..9a0427e1fc7 100644 --- a/go/viperutil/v2/funcs/get.go +++ b/go/viperutil/v2/funcs/get.go @@ -6,6 +6,8 @@ import ( "github.com/spf13/viper" ) +// GetPath returns a GetFunc that expands a slice of strings into individual +// paths based on standard POSIX shell $PATH separator parsing. func GetPath(v *viper.Viper) func(key string) []string { return func(key string) (paths []string) { for _, val := range v.GetStringSlice(key) { diff --git a/go/viperutil/v2/funcs/get_test.go b/go/viperutil/v2/funcs/get_test.go new file mode 100644 index 00000000000..4d68307b026 --- /dev/null +++ b/go/viperutil/v2/funcs/get_test.go @@ -0,0 +1,33 @@ +package funcs_test + +import ( + "fmt" + + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil/v2/funcs" + "vitess.io/vitess/go/viperutil/v2/internal/value" +) + +func ExampleGetPath() { + v := viper.New() + + val := viperutil.Configure("path", viperutil.Options[[]string]{ + GetFunc: funcs.GetPath, + }) + + stub(val, v) + + v.Set(val.Key(), []string{"/var/www", "/usr:/usr/bin", "/vt"}) + fmt.Println(val.Get()) + // Output: [/var/www /usr /usr/bin /vt] +} + +func stub[T any](val viperutil.Value[T], v *viper.Viper) { + // N.B.: You cannot do this in normal code because these types are internal + // to viperutil, but you also will not need to do this. However it's + // necessary for the example to work here. + base := val.(*value.Static[T]).Base + base.BoundGetFunc = base.GetFunc(v) +} From cabbc1340d3703bd47ab22e479ba2b7457f8084b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 12 Nov 2022 06:42:33 -0500 Subject: [PATCH 034/100] copyright headers Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 16 ++++++++++++++++ go/viperutil/v2/config_test.go | 16 ++++++++++++++++ go/viperutil/v2/debug/debug.go | 17 +++++++++++++++++ go/viperutil/v2/errors.go | 16 ++++++++++++++++ go/viperutil/v2/funcs/decode.go | 16 ++++++++++++++++ go/viperutil/v2/funcs/get.go | 16 ++++++++++++++++ go/viperutil/v2/funcs/get_test.go | 16 ++++++++++++++++ go/viperutil/v2/get_func.go | 16 ++++++++++++++++ go/viperutil/v2/internal/log/log.go | 16 ++++++++++++++++ go/viperutil/v2/internal/registry/registry.go | 16 ++++++++++++++++ go/viperutil/v2/internal/sync/sync.go | 16 ++++++++++++++++ go/viperutil/v2/internal/sync/sync_test.go | 16 ++++++++++++++++ go/viperutil/v2/internal/value/value.go | 16 ++++++++++++++++ go/viperutil/v2/value.go | 16 ++++++++++++++++ go/viperutil/v2/viper.go | 16 ++++++++++++++++ go/viperutil/v2/viper_test.go | 16 ++++++++++++++++ go/viperutil/v2/vipertest/stub.go | 16 ++++++++++++++++ 17 files changed, 273 insertions(+) diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go index 1c15f05ff26..f6a0253c70b 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/v2/config.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/config_test.go b/go/viperutil/v2/config_test.go index 19a3b3c6754..00d4bc64eda 100644 --- a/go/viperutil/v2/config_test.go +++ b/go/viperutil/v2/config_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/debug/debug.go b/go/viperutil/v2/debug/debug.go index 47ae495f35b..fb918b7e95b 100644 --- a/go/viperutil/v2/debug/debug.go +++ b/go/viperutil/v2/debug/debug.go @@ -1,7 +1,24 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package debug import ( "github.com/spf13/viper" + "vitess.io/vitess/go/viperutil/v2/internal/registry" ) diff --git a/go/viperutil/v2/errors.go b/go/viperutil/v2/errors.go index f1ebd91e41f..9f363348327 100644 --- a/go/viperutil/v2/errors.go +++ b/go/viperutil/v2/errors.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/funcs/decode.go b/go/viperutil/v2/funcs/decode.go index 1816655635c..578dd6397e1 100644 --- a/go/viperutil/v2/funcs/decode.go +++ b/go/viperutil/v2/funcs/decode.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package funcs // TODO: this creates an import cycle ... IMO tabletenv should not define Seconds, diff --git a/go/viperutil/v2/funcs/get.go b/go/viperutil/v2/funcs/get.go index 9a0427e1fc7..198ab11edcf 100644 --- a/go/viperutil/v2/funcs/get.go +++ b/go/viperutil/v2/funcs/get.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package funcs import ( diff --git a/go/viperutil/v2/funcs/get_test.go b/go/viperutil/v2/funcs/get_test.go index 4d68307b026..b6008fa062f 100644 --- a/go/viperutil/v2/funcs/get_test.go +++ b/go/viperutil/v2/funcs/get_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package funcs_test import ( diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index 7d4cd1c6023..36d1288d534 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/internal/log/log.go b/go/viperutil/v2/internal/log/log.go index 4484a63fd2f..5b0a9178045 100644 --- a/go/viperutil/v2/internal/log/log.go +++ b/go/viperutil/v2/internal/log/log.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package log import ( diff --git a/go/viperutil/v2/internal/registry/registry.go b/go/viperutil/v2/internal/registry/registry.go index fdac6e857a1..afe1dd7b30b 100644 --- a/go/viperutil/v2/internal/registry/registry.go +++ b/go/viperutil/v2/internal/registry/registry.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package registry import ( diff --git a/go/viperutil/v2/internal/sync/sync.go b/go/viperutil/v2/internal/sync/sync.go index 4c064f2cc84..92fcceeaeea 100644 --- a/go/viperutil/v2/internal/sync/sync.go +++ b/go/viperutil/v2/internal/sync/sync.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sync import ( diff --git a/go/viperutil/v2/internal/sync/sync_test.go b/go/viperutil/v2/internal/sync/sync_test.go index 9382e5709c4..32fef4dc4f1 100644 --- a/go/viperutil/v2/internal/sync/sync_test.go +++ b/go/viperutil/v2/internal/sync/sync_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package sync_test import ( diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/v2/internal/value/value.go index 36c441d044a..a540668c346 100644 --- a/go/viperutil/v2/internal/value/value.go +++ b/go/viperutil/v2/internal/value/value.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package value import ( diff --git a/go/viperutil/v2/value.go b/go/viperutil/v2/value.go index fadd26611de..1f816e4c657 100644 --- a/go/viperutil/v2/value.go +++ b/go/viperutil/v2/value.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/viper.go b/go/viperutil/v2/viper.go index 6e6d2d3a073..0e999a6b51d 100644 --- a/go/viperutil/v2/viper.go +++ b/go/viperutil/v2/viper.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/viper_test.go b/go/viperutil/v2/viper_test.go index 4dfdc683eae..60cf1bd9883 100644 --- a/go/viperutil/v2/viper_test.go +++ b/go/viperutil/v2/viper_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package viperutil import ( diff --git a/go/viperutil/v2/vipertest/stub.go b/go/viperutil/v2/vipertest/stub.go index 51921443b29..fe5cd945fde 100644 --- a/go/viperutil/v2/vipertest/stub.go +++ b/go/viperutil/v2/vipertest/stub.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package vipertest import ( From 9c20ba9f83b846c0696c0c9d605b306c21f18361 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 12 Nov 2022 06:48:44 -0500 Subject: [PATCH 035/100] finish switch case for handling config-file finding errors Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go index f6a0253c70b..7e864b87a02 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/v2/config.go @@ -120,12 +120,18 @@ func LoadConfig() error { if err != nil { if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { + msg := "Failed to read in config %s: %s" switch configFileNotFoundHandling.Get() { + case WarnOnConfigFileNotFound: + log.WARN(msg, registry.Static.ConfigFileUsed(), nferr.Error()) + fallthrough // after warning, ignore the error case IgnoreConfigFileNotFound: - // TODO: finish this switch block. + err = nil + case ErrorOnConfigFileNotFound: + log.ERROR(msg, registry.Static.ConfigFileUsed(), nferr.Error()) + case ExitOnConfigFileNotFound: + log.CRITICAL(msg, registry.Static.ConfigFileUsed(), nferr.Error()) } - log.WARN("Failed to read in config %s: %s", registry.Static.ConfigFileUsed(), nferr.Error()) - err = nil } } From 5b48d5f46d4c40651d2eeb6d8944d48785cf3122 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 12 Nov 2022 10:04:52 -0500 Subject: [PATCH 036/100] document sync subpackage Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 9 ++++++ go/viperutil/v2/internal/sync/sync.go | 42 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go index 7e864b87a02..dbbc65b2823 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/v2/config.go @@ -142,6 +142,15 @@ func LoadConfig() error { return registry.Dynamic.Watch(registry.Static) } +// NotifyConfigReload adds a subscription that the dynamic registry will attempt +// to notify on config changes. The notification fires after the updated config +// has been loaded from disk into the live config. +// +// Analogous to signal.Notify, notifications are sent non-blocking, so users +// should account for this when writing code to consume from the channel. +// +// This function must be called prior to LoadConfig; it will panic if called +// after the dynamic registry has started watching the loaded config. func NotifyConfigReload(ch chan<- struct{}) { registry.Dynamic.Notify(ch) } diff --git a/go/viperutil/v2/internal/sync/sync.go b/go/viperutil/v2/internal/sync/sync.go index 92fcceeaeea..22d51c4afde 100644 --- a/go/viperutil/v2/internal/sync/sync.go +++ b/go/viperutil/v2/internal/sync/sync.go @@ -26,6 +26,14 @@ import ( "github.com/spf13/viper" ) +// Viper is a wrapper around a pair of viper.Viper instances to provide config- +// reloading in a threadsafe manner. +// +// It maintains one viper, called "disk", which does the actual config watch and +// reload (via viper's WatchConfig), and a second viper, called "live", which +// Values (registered via viperutil.Configure with Dynamic=true) access their +// settings from. The "live" config only updates after blocking all values from +// reading in order to swap in the most recently-loaded config from the "disk". type Viper struct { m sync.Mutex // prevents races between loadFromDisk and AllSettings disk *viper.Viper @@ -36,6 +44,7 @@ type Viper struct { watchingConfig bool } +// New returns a new synced Viper. func New() *Viper { return &Viper{ disk: viper.New(), @@ -44,8 +53,23 @@ func New() *Viper { } } +// ErrDuplicateWatch is returned when Watch is called on a synced Viper which +// has already started a watch. var ErrDuplicateWatch = errors.New("duplicate watch") +// Watch starts watching the config used by the passed-in Viper. Before starting +// the watch, the synced viper will perform an initial read and load from disk +// so that the live config is ready for use without requiring an initial config +// change. +// +// If the given static viper did not load a config file (and is instead relying +// purely on defaults, flags, and environment variables), then the settings of +// that viper are merged over, and this synced Viper may be used to set up an +// actual watch later. +// +// If this synced viper is already watching a config file, this function returns +// an ErrDuplicateWatch. Other errors may be returned via underlying viper code +// to ensure the config file can be read in properly. func (v *Viper) Watch(static *viper.Viper) error { if v.watchingConfig { return fmt.Errorf("%w: viper is already watching %s", ErrDuplicateWatch, v.disk.ConfigFileUsed()) @@ -85,6 +109,15 @@ func (v *Viper) Watch(static *viper.Viper) error { return nil } +// Notify adds a subscription that this synced viper will attempt to notify on +// config changes, after the updated config has been copied over from disk to +// live. +// +// Analogous to signal.Notify, notifications are sent non-blocking, so users +// should account for this when consuming from the channel they've provided. +// +// This function must be called prior to setting up a Watch; it will panic if a +// a watch has already been established on this synced Viper. func (v *Viper) Notify(ch chan<- struct{}) { if v.watchingConfig { panic("cannot Notify after starting to watch a config") @@ -93,6 +126,7 @@ func (v *Viper) Notify(ch chan<- struct{}) { v.subscribers = append(v.subscribers, ch) } +// AllSettings returns the current live settings. func (v *Viper) AllSettings() map[string]any { v.m.Lock() defer v.m.Unlock() @@ -119,6 +153,14 @@ func (v *Viper) SetDefault(key string, value any) { v.live.SetDefaul // end implementation of registry.Bindable for sync.Viper +// AdaptGetter wraps a get function (matching the signature of +// viperutil.Options.GetFunc) to be threadsafe with the passed-in synced Viper. +// +// It must be called prior to starting a watch on the synced Viper; it will +// panic if a watch has already been established. +// +// This function must be called at most once per key; it will panic if attempting +// to adapt multiple getters for the same key. func AdaptGetter[T any](key string, getter func(v *viper.Viper) func(key string) T, v *Viper) func(key string) T { if v.watchingConfig { panic("cannot adapt getter to synchronized viper which is already watching a config") From 383ab1030c1d412e2b21641784f512d27fc0f5c1 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 12 Nov 2022 15:49:28 -0500 Subject: [PATCH 037/100] document config stuff Signed-off-by: Andrew Mason --- go/viperutil/v2/config.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/go/viperutil/v2/config.go b/go/viperutil/v2/config.go index dbbc65b2823..ad59687a8a9 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/v2/config.go @@ -84,6 +84,11 @@ func init() { registry.Static.SetDefault(configPaths.Key(), configPaths.Default()) } +// RegisterFlags installs the flags that control viper config-loading behavior. +// It is exported to be called by servenv before parsing flags for all binaries. +// +// It cannot be registered here via servenv.OnParse since this causes an import +// cycle. func RegisterFlags(fs *pflag.FlagSet) { fs.StringSlice("config-path", configPaths.Default(), "Paths to search for config files in.") fs.String("config-type", configType.Default(), "Config file type (omit to infer config type from file extension).") @@ -96,6 +101,25 @@ func RegisterFlags(fs *pflag.FlagSet) { BindFlags(fs, configPaths, configType, configName, configFile) } +// LoadConfig attempts to find, and then load, a config file for viper-backed +// config values to use. +// +// Config searching follows the behavior used by viper [1], namely: +// * --config-file (full path, including extension) if set will be used to the +// exclusion of all other flags. +// * --config-type is required if the config file does not have one of viper's +// supported extensions (.yaml, .yml, .json, and so on) +// +// An additional --config-file-not-found-handling flag controls how to treat the +// situation where viper cannot find any config files in any of the provided +// paths (for ex, users may want to exit immediately if a config file that +// should exist doesn't for some reason, or may wish to operate with flags and +// environment variables alone, and not use config files at all). +// +// If a config file is successfully loaded, then the dynamic registry will also +// start watching that file for changes. +// +// [1]: https://github.com/spf13/viper#reading-config-files. func LoadConfig() error { var err error switch file := configFile.Get(); file { @@ -155,12 +179,24 @@ func NotifyConfigReload(ch chan<- struct{}) { registry.Dynamic.Notify(ch) } +// ConfigFileNotFoundHandling is an enum to control how LoadConfig treats errors +// of type viper.ConfigFileNotFoundError when loading a config. type ConfigFileNotFoundHandling int const ( + // IgnoreConfigFileNotFound causes LoadConfig to completely ignore a + // ConfigFileNotFoundError (i.e. not even logging it). IgnoreConfigFileNotFound ConfigFileNotFoundHandling = iota + // WarnOnConfigFileNotFound causes LoadConfig to log a warning with details + // about the failed config load, but otherwise proceeds with the given + // process, which will get config values entirely from defaults, + // envirnoment variables, and flags. WarnOnConfigFileNotFound + // ErrorOnConfigFileNotFound causes LoadConfig to return the + // ConfigFileNotFoundError after logging an error. ErrorOnConfigFileNotFound + // ExitOnConfigFileNotFound causes LoadConfig to log.Fatal on a + // ConfigFileNotFoundError. ExitOnConfigFileNotFound ) From 1b0aafb61a3481f1776228b08206769a1c1d3b4f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 13 Nov 2022 09:44:34 -0500 Subject: [PATCH 038/100] more docs Signed-off-by: Andrew Mason --- go/viperutil/v2/errors.go | 8 +++++++- go/viperutil/v2/value.go | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/go/viperutil/v2/errors.go b/go/viperutil/v2/errors.go index 9f363348327..eddeb699332 100644 --- a/go/viperutil/v2/errors.go +++ b/go/viperutil/v2/errors.go @@ -22,6 +22,12 @@ import ( ) var ( + // ErrDuplicateWatch is returned when Watch is called multiple times on a + // single synced viper. Viper only supports reading/watching a single + // config file. ErrDuplicateWatch = sync.ErrDuplicateWatch - ErrNoFlagDefined = value.ErrNoFlagDefined + // ErrNoFlagDefined is returned from Value's Flag method when the value was + // configured to bind to a given FlagName but the provided flag set does not + // define a flag with that name. + ErrNoFlagDefined = value.ErrNoFlagDefined ) diff --git a/go/viperutil/v2/value.go b/go/viperutil/v2/value.go index 1f816e4c657..1ca88b0737a 100644 --- a/go/viperutil/v2/value.go +++ b/go/viperutil/v2/value.go @@ -27,12 +27,29 @@ var ( _ Value[int] = (*value.Dynamic[int])(nil) ) +// Value represents the public API to access viper-backed config values. +// +// N.B. the embedded value.Registerable interface is necessary only for +// BindFlags and other mechanisms of binding Values to the internal registries +// to work. Users of Value objects should only need to call Get() and Default(). type Value[T any] interface { value.Registerable + + // Get returns the current value. For static implementations, this will + // never change after the initial config load. For dynamic implementations, + // this may change throughout the lifetime of the vitess process. Get() T + // Default returns the default value configured for this Value. For both + // static and dynamic implementations, it should never change. Default() T } +// BindFlags binds a set of Registerable values to the given flag set. +// +// This function will panic if any of the values was configured to map to a flag +// which is not defined on the flag set. Therefore, this function should usually +// be called in an OnParse or OnParseFor hook after defining the flags for the +// values in question. func BindFlags(fs *pflag.FlagSet, values ...value.Registerable) { value.BindFlags(fs, values...) } From c9ea5ccbb6eec3b29d055b34e2e184b61d7a0b2f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 13 Nov 2022 09:57:41 -0500 Subject: [PATCH 039/100] document internal/log Signed-off-by: Andrew Mason --- go/viperutil/v2/internal/log/log.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/go/viperutil/v2/internal/log/log.go b/go/viperutil/v2/internal/log/log.go index 5b0a9178045..28a03d17289 100644 --- a/go/viperutil/v2/internal/log/log.go +++ b/go/viperutil/v2/internal/log/log.go @@ -14,6 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package log provides dual-use logging between vitess's vt/log package and +viper's jww log. +*/ package log import ( @@ -37,10 +41,17 @@ var ( } } - TRACE = jwwlog(jww.TRACE, nil) - DEBUG = jwwlog(jww.DEBUG, nil) - INFO = jwwlog(jww.INFO, log.Infof) - WARN = jwwlog(jww.WARN, log.Warningf) - ERROR = jwwlog(jww.ERROR, log.Errorf) + // TRACE logs to viper's TRACE level, and nothing to vitess logs. + TRACE = jwwlog(jww.TRACE, nil) + // DEBUG logs to viper's DEBUG level, and nothing to vitess logs. + DEBUG = jwwlog(jww.DEBUG, nil) + // INFO logs to viper and vitess at INFO levels. + INFO = jwwlog(jww.INFO, log.Infof) + // WARN logs to viper and vitess at WARN/WARNING levels. + WARN = jwwlog(jww.WARN, log.Warningf) + // ERROR logs to viper and vitess at ERROR levels. + ERROR = jwwlog(jww.ERROR, log.Errorf) + // CRITICAL logs to viper at CRITICAL level, and then fatally logs to + // vitess, exiting the process. CRITICAL = jwwlog(jww.CRITICAL, log.Fatalf) ) From f33c2494061cdc42df30f2ca1621cb012c50c056 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 15 Nov 2022 06:49:02 -0500 Subject: [PATCH 040/100] document Options and Configure Signed-off-by: Andrew Mason --- go/viperutil/v2/viper.go | 64 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/go/viperutil/v2/viper.go b/go/viperutil/v2/viper.go index 0e999a6b51d..05ce3316fbe 100644 --- a/go/viperutil/v2/viper.go +++ b/go/viperutil/v2/viper.go @@ -24,17 +24,64 @@ import ( "vitess.io/vitess/go/viperutil/v2/internal/value" ) +// Options represents the various options used to control how Values are +// configured by viperutil. type Options[T any] struct { - Aliases []string + // Aliases, if set, configures the Value to be accessible via additional + // keys. + // + // This is useful for deprecating old names gracefully while maintaining + // backwards-compatibility. + Aliases []string + // FlagName, if set, allows a value to be configured to also check the + // named flag for its final config value. If depending on a flag, BindFlags + // must be called on the Value returned from Configure. In most cases, + // modules will do this in the same OnParse hook that defines their flags. + // + // Note that if the set FlagName does not match the Value's key, Configure + // will automatically register an alias to allow both names to be used as + // the key, which is necessary for the flag value to be discoverable by + // viper for the Value's actual key. FlagName string - EnvVars []string - Default T - + // EnvVars, if set, configures the Value to also check the given environment + // variables for its final config value. + // + // Note that unlike keys and aliases, environment variable names are + // case-sensitive. + EnvVars []string + // Default is the default value that will be set for the key. If not + // explicitly set during a call to Configure, the default value will be the + // zero value for the type T. This means if T is a pointer type, the default + // will be nil, not the zeroed out struct. + Default T + + // Dynamic, if set, configures a value to be backed by the dynamic registry. + // If a config file is used (via LoadConfig), that file will be watched for + // changes, and dynamic Values will reflect changes via their Get() methods + // (whereas static values will only ever return the value loaded initially). Dynamic bool + // GetFunc is the function used to get this value out of a viper. + // + // If omitted, GetFuncForType will attempt to provide a useful default for + // the given type T. For primitive types, this should be sufficient. For + // more fine-grained control over value retrieval and unmarshalling, callers + // should provide their own function. + // + // See GetFuncForType for further details. GetFunc func(v *viper.Viper) func(key string) T } +// Configure configures a viper-backed value associated with the given key to +// either the static or dynamic internal registries, returning the resulting +// Value. This value is partially ready for use (it will be able to get values +// from environment variables and defaults), but file-based configs will not +// available until servenv calls LoadConfig, and flag-based configs will not be +// available until a combination of BindFlags and pflag.Parse have been called, +// usually by servenv. +// +// Exact behavior of how the key is bound to the registries depend on the +// Options provided, func Configure[T any](key string, opts Options[T]) (v Value[T]) { getfunc := opts.GetFunc if getfunc == nil { @@ -60,6 +107,15 @@ func Configure[T any](key string, opts Options[T]) (v Value[T]) { return v } +// KeyPrefixFunc is a helper function to allow modules to extract a common key +// prefix used by that module to avoid repitition (and typos, missed updates, +// and so on). +// +// For example, package go/vt/vttablet/schema may want to do: +// +// moduleKey := viperutil.KeyPrefixFunc("vttablet.schema") +// watch := viperutil.Configure(moduleKey("watch_interval"), ...) // => "vttablet.schema.watch_interval" +// // ... and so on func KeyPrefixFunc(prefix string) func(subkey string) (fullkey string) { var keyParts []string if prefix != "" { From 35b4328da244b13ab903c7735e0d84e801aa1999 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 15 Nov 2022 06:50:31 -0500 Subject: [PATCH 041/100] rename Signed-off-by: Andrew Mason --- go/viperutil/v2/{viper_test.go => get_func_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename go/viperutil/v2/{viper_test.go => get_func_test.go} (100%) diff --git a/go/viperutil/v2/viper_test.go b/go/viperutil/v2/get_func_test.go similarity index 100% rename from go/viperutil/v2/viper_test.go rename to go/viperutil/v2/get_func_test.go From 59c7c4c77a049b98b0b14042df5dd383ddee3e27 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 15 Nov 2022 09:26:29 -0500 Subject: [PATCH 042/100] more test cases for GetFuncForType Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func_test.go | 46 +++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/v2/get_func_test.go index 60cf1bd9883..9a5aff8088f 100644 --- a/go/viperutil/v2/get_func_test.go +++ b/go/viperutil/v2/get_func_test.go @@ -25,12 +25,46 @@ import ( func TestGetFuncForType(t *testing.T) { v := viper.New() - v.Set("foo", true) - v.Set("bar", 5) - v.Set("baz", []int{1, 2, 3}) + v.Set("foo.bool", true) + v.Set("foo.int", 5) + v.Set("foo.float", 5.1) + v.Set("foo.intslice", []int{1, 2, 3}) + v.Set("foo.stringslice", []string{"a", "b", "c"}) + v.Set("foo.string", "hello") - getBool := GetFuncForType[bool]() - assert.True(t, getBool(v)("foo")) + assert := assert.New(t) - assert.Equal(t, 5, GetFuncForType[int]()(v)("bar")) + // Bool types + assert.Equal(true, get[bool](t, v, "foo.bool"), "GetFuncForType[bool](foo.bool)") + + // Int types + assert.Equal(5, get[int](t, v, "foo.int"), "GetFuncForType[int](foo.int)") + assert.Equal(int8(5), get[int8](t, v, "foo.int"), "GetFuncForType[int8](foo.int)") + assert.Equal(int16(5), get[int16](t, v, "foo.int"), "GetFuncForType[int16](foo.int)") + assert.Equal(int32(5), get[int32](t, v, "foo.int"), "GetFuncForType[int32](foo.int)") + assert.Equal(int64(5), get[int64](t, v, "foo.int"), "GetFuncForType[int64](foo.int)") + + // Uint types + assert.Equal(uint(5), get[uint](t, v, "foo.int"), "GetFuncForType[uint](foo.int)") + assert.Equal(uint8(5), get[uint8](t, v, "foo.int"), "GetFuncForType[uint8](foo.int)") + assert.Equal(uint16(5), get[uint16](t, v, "foo.int"), "GetFuncForType[uint16](foo.int)") + assert.Equal(uint32(5), get[uint32](t, v, "foo.int"), "GetFuncForType[uint32](foo.int)") + assert.Equal(uint64(5), get[uint64](t, v, "foo.int"), "GetFuncForType[uint64](foo.int)") + + // Float types + assert.Equal(5.1, get[float64](t, v, "foo.float"), "GetFuncForType[float64](foo.float)") + assert.Equal(float32(5.1), get[float32](t, v, "foo.float"), "GetFuncForType[float32](foo.float)") + assert.Equal(float64(5), get[float64](t, v, "foo.int"), "GetFuncForType[float64](foo.int)") + + // Slice types + assert.ElementsMatch([]int{1, 2, 3}, get[[]int](t, v, "foo.intslice"), "GetFuncForType[[]int](foo.intslice)") + assert.ElementsMatch([]string{"a", "b", "c"}, get[[]string](t, v, "foo.stringslice"), "GetFuncForType[[]string](foo.stringslice)") + + // String types + assert.Equal("hello", get[string](t, v, "foo.string"), "GetFuncForType[string](foo.string)") +} + +func get[T any](t testing.TB, v *viper.Viper, key string) T { + t.Helper() + return GetFuncForType[T]()(v)(key) } From 2c77187e0561d0d1cdc7fa6d8248a5b8d73b4dec Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 15 Nov 2022 09:40:38 -0500 Subject: [PATCH 043/100] more cases Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func.go | 3 +++ go/viperutil/v2/get_func_test.go | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index 36d1288d534..42fc3985fc3 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -76,6 +76,7 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { return v.GetUint64 } case reflect.Uintptr: + // Unupported, fallthrough to `if f == nil` check below switch. case reflect.Float32: f = func(v *viper.Viper) func(key string) float32 { return func(key string) float32 { @@ -90,7 +91,9 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { case reflect.Complex128: case reflect.Array: case reflect.Chan: + panic("GetFuncForType does not support channel types") case reflect.Func: + panic("GetFuncForType does not support function types") case reflect.Interface: // TODO: unwrap to see if struct-ish case reflect.Map: diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/v2/get_func_test.go index 9a5aff8088f..74128d93754 100644 --- a/go/viperutil/v2/get_func_test.go +++ b/go/viperutil/v2/get_func_test.go @@ -18,19 +18,24 @@ package viperutil import ( "testing" + "time" "github.com/spf13/viper" "github.com/stretchr/testify/assert" ) func TestGetFuncForType(t *testing.T) { + now := time.Now() + v := viper.New() v.Set("foo.bool", true) v.Set("foo.int", 5) + v.Set("foo.duration", time.Second) v.Set("foo.float", 5.1) v.Set("foo.intslice", []int{1, 2, 3}) v.Set("foo.stringslice", []string{"a", "b", "c"}) v.Set("foo.string", "hello") + v.Set("foo.time", now) assert := assert.New(t) @@ -44,6 +49,9 @@ func TestGetFuncForType(t *testing.T) { assert.Equal(int32(5), get[int32](t, v, "foo.int"), "GetFuncForType[int32](foo.int)") assert.Equal(int64(5), get[int64](t, v, "foo.int"), "GetFuncForType[int64](foo.int)") + // Duration types + assert.Equal(time.Second, get[time.Duration](t, v, "foo.duration"), "GetFuncForType[time.Duration](foo.duration)") + // Uint types assert.Equal(uint(5), get[uint](t, v, "foo.int"), "GetFuncForType[uint](foo.int)") assert.Equal(uint8(5), get[uint8](t, v, "foo.int"), "GetFuncForType[uint8](foo.int)") @@ -62,6 +70,9 @@ func TestGetFuncForType(t *testing.T) { // String types assert.Equal("hello", get[string](t, v, "foo.string"), "GetFuncForType[string](foo.string)") + + // Struct types + assert.Equal(now, get[time.Time](t, v, "foo.time"), "GetFuncForType[time.Time](foo.time)") } func get[T any](t testing.TB, v *viper.Viper, key string) T { From 3c67de2e4a45b834c384eda2a121532eff807848 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 15 Nov 2022 15:26:04 -0500 Subject: [PATCH 044/100] struct support Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func.go | 11 +++++++---- go/viperutil/v2/get_func_test.go | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index 42fc3985fc3..e06a0b9b6e1 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -118,7 +118,10 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { } } case reflect.Pointer: - // TODO: unwrap to see if struct-ish + switch typ.Elem().Kind() { + case reflect.Struct: + f = unmarshalFunc[T]() + } case reflect.Slice: switch typ.Elem().Kind() { case reflect.Int: @@ -159,10 +162,10 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { func unmarshalFunc[T any]() func(v *viper.Viper) func(key string) T { return func(v *viper.Viper) func(key string) T { - var t T return func(key string) T { - _ = v.UnmarshalKey(key, t) - return t + t := new(T) + _ = v.UnmarshalKey(key, t) // TODO: panic on this error + return *t } } } diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/v2/get_func_test.go index 74128d93754..d4829edc25b 100644 --- a/go/viperutil/v2/get_func_test.go +++ b/go/viperutil/v2/get_func_test.go @@ -24,6 +24,16 @@ import ( "github.com/stretchr/testify/assert" ) +type myStruct struct { + Foo string + Bar int +} + +type myNestedStruct struct { + MyStruct *myStruct + Baz bool +} + func TestGetFuncForType(t *testing.T) { now := time.Now() @@ -73,6 +83,29 @@ func TestGetFuncForType(t *testing.T) { // Struct types assert.Equal(now, get[time.Time](t, v, "foo.time"), "GetFuncForType[time.Time](foo.time)") + { + s := &myStruct{ + Foo: "hello", + Bar: 3, + } + v.Set("mystruct.foo", s.Foo) + v.Set("mystruct.bar", s.Bar) + + assert.Equal(s, get[*myStruct](t, v, "mystruct"), "GetFuncForType[*myStruct](mystruct)") + assert.IsType(&myStruct{}, get[*myStruct](t, v, "mystruct"), "GetFuncForType[*myStruct](mystruct) should return a pointer") + assert.Equal(*s, get[myStruct](t, v, "mystruct"), "GetFuncForType[myStruct](mystruct)") + assert.IsType(myStruct{}, get[myStruct](t, v, "mystruct"), "GetFuncForType[myStruct](mystruct) should return a struct (not pointer to struct)") + + s2 := &myNestedStruct{ + MyStruct: s, + Baz: true, + } + v.Set("mynestedstruct.mystruct.foo", s2.MyStruct.Foo) + v.Set("mynestedstruct.mystruct.bar", s2.MyStruct.Bar) + v.Set("mynestedstruct.baz", s2.Baz) + assert.Equal(*s2, get[myNestedStruct](t, v, "mynestedstruct"), "GetFuncForType[myNestedStruct](mynestedstruct)") + assert.IsType(myNestedStruct{}, get[myNestedStruct](t, v, "mynestedstruct"), "GetFuncForType[myNestedStruct](mynestedstruct) should return a struct (not pointer to struct)") + } } func get[T any](t testing.TB, v *viper.Viper, key string) T { From 0574e0dcf7ddc3020a8efffe0ce2a3f76d4c193e Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 15 Nov 2022 15:30:52 -0500 Subject: [PATCH 045/100] support complex types Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func.go | 16 ++++++++++++++++ go/viperutil/v2/get_func_test.go | 10 ++++++++++ 2 files changed, 26 insertions(+) diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index e06a0b9b6e1..e4c3b9c6b70 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -19,6 +19,7 @@ package viperutil import ( "fmt" "reflect" + "strconv" "time" "github.com/spf13/viper" @@ -88,7 +89,9 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { return v.GetFloat64 } case reflect.Complex64: + f = getComplex[complex64](64) case reflect.Complex128: + f = getComplex[complex128](128) case reflect.Array: case reflect.Chan: panic("GetFuncForType does not support channel types") @@ -185,3 +188,16 @@ func getCastedUint[T uint8 | uint16]() func(v *viper.Viper) func(key string) T { } } } + +func getComplex[T complex64 | complex128](bitSize int) func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + x, err := strconv.ParseComplex(v.GetString(key), bitSize) + if err != nil { + panic(err) // TODO: wrap with more details (key, type (64 vs 128), etc) + } + + return T(x) + } + } +} diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/v2/get_func_test.go index d4829edc25b..390c429248b 100644 --- a/go/viperutil/v2/get_func_test.go +++ b/go/viperutil/v2/get_func_test.go @@ -17,6 +17,7 @@ limitations under the License. package viperutil import ( + "fmt" "testing" "time" @@ -42,6 +43,7 @@ func TestGetFuncForType(t *testing.T) { v.Set("foo.int", 5) v.Set("foo.duration", time.Second) v.Set("foo.float", 5.1) + v.Set("foo.complex", fmt.Sprintf("%v", complex(1, 2))) v.Set("foo.intslice", []int{1, 2, 3}) v.Set("foo.stringslice", []string{"a", "b", "c"}) v.Set("foo.string", "hello") @@ -74,6 +76,14 @@ func TestGetFuncForType(t *testing.T) { assert.Equal(float32(5.1), get[float32](t, v, "foo.float"), "GetFuncForType[float32](foo.float)") assert.Equal(float64(5), get[float64](t, v, "foo.int"), "GetFuncForType[float64](foo.int)") + // Complex types + assert.Equal(complex(5, 0), get[complex128](t, v, "foo.int"), "GetFuncForType[complex128](foo.int)") + assert.Equal(complex(5.1, 0), get[complex128](t, v, "foo.float"), "GetFuncForType[complex128](foo.float)") + assert.Equal(complex(1, 2), get[complex128](t, v, "foo.complex"), "GetFuncForType[complex128](foo.complex)") + assert.Equal(complex64(complex(5, 0)), get[complex64](t, v, "foo.int"), "GetFuncForType[complex64](foo.int)") + assert.Equal(complex64(complex(5.1, 0)), get[complex64](t, v, "foo.float"), "GetFuncForType[complex64](foo.float)") + assert.Equal(complex64(complex(1, 2)), get[complex64](t, v, "foo.complex"), "GetFuncForType[complex64](foo.complex)") + // Slice types assert.ElementsMatch([]int{1, 2, 3}, get[[]int](t, v, "foo.intslice"), "GetFuncForType[[]int](foo.intslice)") assert.ElementsMatch([]string{"a", "b", "c"}, get[[]string](t, v, "foo.stringslice"), "GetFuncForType[[]string](foo.stringslice)") From fe994d461b29732ee041d37a648836d36a0c0a12 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 16 Nov 2022 06:47:04 -0500 Subject: [PATCH 046/100] map types, realizing we cannot support interfaces Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func.go | 2 +- go/viperutil/v2/get_func_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index e4c3b9c6b70..9b6669db288 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -98,7 +98,7 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { case reflect.Func: panic("GetFuncForType does not support function types") case reflect.Interface: - // TODO: unwrap to see if struct-ish + panic("GetFuncForType does not support interface types (specify a specific implementation type instead)") case reflect.Map: switch typ.Key().Kind() { case reflect.String: diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/v2/get_func_test.go index 390c429248b..50d712e59b6 100644 --- a/go/viperutil/v2/get_func_test.go +++ b/go/viperutil/v2/get_func_test.go @@ -18,6 +18,7 @@ package viperutil import ( "fmt" + "strings" "testing" "time" @@ -116,6 +117,26 @@ func TestGetFuncForType(t *testing.T) { assert.Equal(*s2, get[myNestedStruct](t, v, "mynestedstruct"), "GetFuncForType[myNestedStruct](mynestedstruct)") assert.IsType(myNestedStruct{}, get[myNestedStruct](t, v, "mynestedstruct"), "GetFuncForType[myNestedStruct](mynestedstruct) should return a struct (not pointer to struct)") } + + // Map types. + v.Set("stringmap", map[string]string{ + "a": "A", + "b": "B", + }) + assert.Equal(map[string]string{"a": "A", "b": "B"}, get[map[string]string](t, v, "stringmap"), "GetFuncForType[map[string]string](stringmap)") + + v.Set("stringslicemap", map[string][]string{ + "uppers": strings.Split("ABCDEFG", ""), + "lowers": strings.Split("abcdefg", ""), + }) + assert.Equal(map[string][]string{"uppers": strings.Split("ABCDEFG", ""), "lowers": strings.Split("abcdefg", "")}, get[map[string][]string](t, v, "stringslicemap"), "GetFuncForType[map[string][]string](stringslicemap)") + + v.Set("anymap", map[string]any{ + "int": 5, + "bool": true, + "string": "hello", + }) + assert.Equal(map[string]any{"int": 5, "bool": true, "string": "hello"}, get[map[string]any](t, v, "anymap"), "GetFuncForType[map[string]any](anymap)") } func get[T any](t testing.TB, v *viper.Viper, key string) T { From 0b5e4c11f0887297dd4a6938cabe3045a2698500 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 16 Nov 2022 10:04:18 -0500 Subject: [PATCH 047/100] finish tests Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func.go | 5 +++++ go/viperutil/v2/get_func_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index 9b6669db288..81d88663bc2 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -93,6 +93,11 @@ func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { case reflect.Complex128: f = getComplex[complex128](128) case reflect.Array: + // Even though the code would be extremely similar to slice types, we + // cannot support arrays because there's no way to write a function that + // returns, say, [N]int, for some value of N which we only know at + // runtime. + panic("GetFuncForType does not support array types") case reflect.Chan: panic("GetFuncForType does not support channel types") case reflect.Func: diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/v2/get_func_test.go index 50d712e59b6..9692ae7b127 100644 --- a/go/viperutil/v2/get_func_test.go +++ b/go/viperutil/v2/get_func_test.go @@ -137,6 +137,36 @@ func TestGetFuncForType(t *testing.T) { "string": "hello", }) assert.Equal(map[string]any{"int": 5, "bool": true, "string": "hello"}, get[map[string]any](t, v, "anymap"), "GetFuncForType[map[string]any](anymap)") + + // Unsupported types. + t.Run("uintptr", func(t *testing.T) { + testPanic(t, GetFuncForType[uintptr], "GetFuncForType[uintptr]") + }) + t.Run("arrays", func(t *testing.T) { + testPanic(t, GetFuncForType[[5]int], "GetFuncForType[[5]int]") + testPanic(t, GetFuncForType[[3]string], "GetFuncForType[[3]string]") + }) + t.Run("channels", func(t *testing.T) { + testPanic(t, GetFuncForType[chan struct{}], "GetFuncForType[chan struct{}]") + testPanic(t, GetFuncForType[chan bool], "GetFuncForType[chan bool]") + testPanic(t, GetFuncForType[chan int], "GetFuncForType[chan int]") + testPanic(t, GetFuncForType[chan chan string], "GetFuncForType[chan chan string]") + }) + t.Run("funcs", func(t *testing.T) { + testPanic(t, GetFuncForType[func()], "GetFuncForType[func()]") + }) +} + +func testPanic[T any](t testing.TB, f func() func(v *viper.Viper) func(key string) T, fnName string) { + t.Helper() + + defer func() { + err := recover() + assert.NotNil(t, err, "%s should panic", fnName) + }() + + fn := f() + assert.Failf(t, fmt.Sprintf("%s should panic", fnName), "%s should panic; got %+v", fnName, fn) } func get[T any](t testing.TB, v *viper.Viper, key string) T { From 0bcbb4f5fdc3b778da72f04d83cb719c26e7ccc1 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 16 Nov 2022 14:30:12 -0500 Subject: [PATCH 048/100] preserve usage for seconds decoder Signed-off-by: Andrew Mason --- go/viperutil/v2/funcs/decode.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/viperutil/v2/funcs/decode.go b/go/viperutil/v2/funcs/decode.go index 578dd6397e1..723df555bcf 100644 --- a/go/viperutil/v2/funcs/decode.go +++ b/go/viperutil/v2/funcs/decode.go @@ -56,3 +56,10 @@ package funcs // s.Set(data.(time.Duration)) // return s, nil // } + +/* USAGE (similar to how ConfigFileNotFoundHandling is decoded in viperutil/config.go) +// viper.DecoderConfigOption(viper.DecodeHook(mapstructure.DecodeHookFunc(decode.Seconds))), +// viper.DecoderConfigOption(func(dc *mapstructure.DecoderConfig) { +// dc +// }), +*/ From 37d42de9b8329d719eba23fd09d58b91ee9ba1da Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 16 Nov 2022 14:30:40 -0500 Subject: [PATCH 049/100] more docs Signed-off-by: Andrew Mason --- go/viperutil/v2/get_func.go | 15 ++++++++++ go/viperutil/v2/viper.go | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/v2/get_func.go index 81d88663bc2..98835cab07f 100644 --- a/go/viperutil/v2/get_func.go +++ b/go/viperutil/v2/get_func.go @@ -25,6 +25,21 @@ import ( "github.com/spf13/viper" ) +// GetFuncForType returns the default getter function for a given type T to. A +// getter function is a function which takes a viper and returns a function that +// takes a key and (finally!) returns a value of type T. +// +// For example, the default getter for a value of type string is a function that +// takes a viper instance v and calls v.GetString with the provided key. +// +// In most cases, callers of Configure should be able to rely on the defaults +// provided here (and may refer to get_func_test.go for an up-to-date example +// of the provided functionalities), but if more fine-grained control is needed +// (this should be an **exceptional** circumstance), they may provide their own +// GetFunc as an option to Configure. +// +// This function may panic if called for an unsupported type. This is captured +// in the test code as well. func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { var ( t T diff --git a/go/viperutil/v2/viper.go b/go/viperutil/v2/viper.go index 05ce3316fbe..c45e8f323d0 100644 --- a/go/viperutil/v2/viper.go +++ b/go/viperutil/v2/viper.go @@ -14,6 +14,65 @@ See the License for the specific language governing permissions and limitations under the License. */ +/* +Package viperutil provides a utility layer to streamline and standardize +interacting with viper-backed configuration values across vitess components. + +The common pattern is for a given module to declare their values by declaring +variables that are the result of calling Configure, for example in package trace: + + package trace + + import "vitess.io/vitess/go/viperutil/v2" + + var ( + modulePrefix = viperutil.KeyPrefixFunc("trace") + + tracingServer = viperutil.Configure( + modulePrefix("service"), + viperutil.Options[string]{ + Default: "noop", + FlagName: "tracer", + } + ) + enableLogging = viperutil.Configure( + modulePrefix("enable-logging"), + viperutil.Options[bool]{ + FlagName: "tracing-enable-logging", + } + ) + ) + +Then, in an OnParseFor or OnParse hook, declare any flags, and bind viper values +to those flags, as appropriate: + + package trace + + import ( + "github.com/spf13/pflag" + + "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/vt/servenv" + ) + + func init() { + servenv.OnParse(func(fs *pflag.FlagSet) { + fs.String("tracer", tracingServer.Default(), "") + fs.Bool("tracing-enable-logging", enableLogging.Default(), "") + + viperutil.BindFlags(fs, tracingServer, enableLogging) + }) + } + +Finally, after a call to `viperutil.LoadConfig` (which is done as a part of +`servenv.ParseFlags`), values may be accessed by calling their `.Get()` methods. + +For more details, refer to the package documentation, as well as the documents +in doc/viper/. + +- TODO: remove original API and `git mv` v2 up a level, adjusting imports +- TODO: markdown doc under doc/ tree giving an overview. +*/ package viperutil import ( From 02c843b9729cdfcfbc541c9ee594bea54ab757df Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 16 Nov 2022 14:35:08 -0500 Subject: [PATCH 050/100] bye bye v1 Signed-off-by: Andrew Mason --- go/viperutil/config.go | 189 ------------- go/viperutil/key.go | 50 ---- go/viperutil/viper.go | 336 ----------------------- go/viperutil/viperget/doc.go | 33 --- go/viperutil/viperget/duration_or_int.go | 41 --- go/viperutil/viperget/path.go | 21 -- go/viperutil/viperget/tablet_type.go | 53 ---- go/viperutil/vipersync/viper.go | 142 ---------- go/viperutil/vipersync/viper_test.go | 143 ---------- 9 files changed, 1008 deletions(-) delete mode 100644 go/viperutil/config.go delete mode 100644 go/viperutil/key.go delete mode 100644 go/viperutil/viper.go delete mode 100644 go/viperutil/viperget/doc.go delete mode 100644 go/viperutil/viperget/duration_or_int.go delete mode 100644 go/viperutil/viperget/path.go delete mode 100644 go/viperutil/viperget/tablet_type.go delete mode 100644 go/viperutil/vipersync/viper.go delete mode 100644 go/viperutil/vipersync/viper_test.go diff --git a/go/viperutil/config.go b/go/viperutil/config.go deleted file mode 100644 index 23a4b4b49e2..00000000000 --- a/go/viperutil/config.go +++ /dev/null @@ -1,189 +0,0 @@ -package viperutil - -import ( - "os" - - jww "github.com/spf13/jwalterweatherman" - "github.com/spf13/pflag" - "github.com/spf13/viper" - - "vitess.io/vitess/go/flagutil" - "vitess.io/vitess/go/viperutil/viperget" - "vitess.io/vitess/go/vt/log" -) - -var ( - v = viper.New() - configPaths = NewValue( - "config.paths", - viperget.Path(v), - WithEnvVars[[]string]("VT_CONFIG_PATH"), - WithFlags[[]string]("config-path"), - ) - - configType = NewValue( - "config.type", - v.GetString, - WithEnvVars[string]("VT_CONFIG_TYPE"), - WithFlags[string]("config-type"), - ) - _configTypeFlagVal = flagutil.NewCaseInsensitiveStringEnum("config-type", configType.Value(), viper.SupportedExts) - - configName = NewValue( - "config.name", - v.GetString, - WithDefault("vtconfig"), - WithEnvVars[string]("VT_CONFIG_NAME"), - WithFlags[string]("config-name"), - ) - configFile = NewValue( - "config.file", - v.GetString, - WithEnvVars[string]("VT_CONFiG_FILE"), - WithFlags[string]("config-file"), - ) -) - -func init() { - wd, _ := os.Getwd() - configPaths.value = append(configPaths.value, wd) -} - -func RegisterDefaultConfigFlags(fs *pflag.FlagSet) { - fs.StringSlice("config-path", configPaths.Value(), "") // TODO: usage here - configPaths.Bind(v, fs) - - fs.Var(_configTypeFlagVal, "config-type", "") // TODO: usage here - configType.Bind(v, fs) - - fs.String("config-name", configName.Value(), "") // TODO: usage here - configName.Bind(v, fs) - - fs.String("config-file", configFile.Value(), "") // TODO: usage here - configFile.Bind(v, fs) -} - -type ConfigFileNotFoundHandling int - -const ( - IgnoreConfigFileNotFound ConfigFileNotFoundHandling = iota - WarnOnConfigFileNotFound - ErrorOnConfigFileNotFound - ExitOnConfigFileNotFound -) - -func (handling ConfigFileNotFoundHandling) String() string { - switch handling { - case IgnoreConfigFileNotFound: - return "ignore" - case WarnOnConfigFileNotFound: - return "warn" - case ErrorOnConfigFileNotFound: - return "error" - case ExitOnConfigFileNotFound: - return "exit" - default: - return "" - } -} - -type ConfigOptions struct { - Paths []string - Type string - Name string - File string -} - -// ReadInConfig attempts to read a config into the given viper instance. -// -// The ConfigOptions govern the config file searching behavior, with Paths, -// Type, Name, and File behaving as described by the viper documentation [1]. -// -// The ConfigFileNotFoundHandling controls whether not finding a config file -// should be ignored, treated as a warning, or exit the program. -// -// [1]: https://pkg.go.dev/github.com/spf13/viper#readme-reading-config-files -func ReadInConfig(v *viper.Viper, opts ConfigOptions, handling ConfigFileNotFoundHandling) error { - for _, path := range opts.Paths { - v.AddConfigPath(path) - } - - if opts.Type != "" { - v.SetConfigType(opts.Type) - } - - if opts.Name != "" { - v.SetConfigName(opts.Name) - } - - if opts.File != "" { - v.SetConfigFile(opts.File) - } - - jww.DEBUG.Printf("Reading in config with options: %+v; FileNotFoundHandling: %s\n", opts, handling.String()) - - err := v.ReadInConfig() - if err != nil { - format, args := "Failed to read in config %s: %s", []any{v.ConfigFileUsed(), err.Error()} - - if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { - switch handling { - case IgnoreConfigFileNotFound: - DEBUG(format, args...) - return nil - case WarnOnConfigFileNotFound: - WARN(format, args...) - log.Warningf(nferr.Error()) - case ErrorOnConfigFileNotFound: - ERROR(format, args...) - return err - case ExitOnConfigFileNotFound: - CRITICAL(format, args...) - log.Exitf(nferr.Error()) - } - } - } - - return err -} - -var ( - jwwlog = func(printer interface { - Printf(format string, args ...any) - }) func(format string, args ...any) { - return printer.Printf - } - - TRACE = jwwlog(jww.TRACE) - DEBUG = jwwlog(jww.DEBUG) - INFO = jwwlog(jww.INFO) - WARN = jwwlog(jww.WARN) - ERROR = jwwlog(jww.ERROR) - CRITICAL = jwwlog(jww.CRITICAL) -) - -// ReadInDefaultConfig reads the default config (governed by the config-* flags -// defined in this package) into the global viper singleton. -// -// It is called by servenv immediately after parsing the command-line flags. -// Modules that manage their own viper instances should merge in this config at -// any point after parsing has occurred (**not** in an init func): -// -// // Assuming this package has initialized a viper as `v := viper.New()`, -// // this merges all settings on the global viper into the local viper. -// v.MergeConfigMap(viper.AllSettings()) -// -// This function will log a warning if a config file cannot be found, since -// users may not necessarily wish to use files to manage their configurations. -func ReadInDefaultConfig() error { - return ReadInConfig( - viper.GetViper(), - ConfigOptions{ - Paths: configPaths.Get(), - Type: configType.Get(), - Name: configName.Get(), - File: configFile.Get(), - }, - WarnOnConfigFileNotFound, - ) -} diff --git a/go/viperutil/key.go b/go/viperutil/key.go deleted file mode 100644 index 4de76210710..00000000000 --- a/go/viperutil/key.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package viperutil - -import "strings" - -// Key returns a full key given a prefix and optional key. If key is the empty -// string, only the prefix is returned, otherwise the prefix is joined with the -// key with ".", the default viper key delimiter. -func Key(prefix string, key string) string { - parts := []string{prefix} - if key != "" { - parts = append(parts, key) - } - - return strings.Join(parts, ".") -} - -// KeyPartial returns a partial func that returns full keys joined to the given -// prefix. -// -// Packages can use this to avoid repeating a prefix across all their key -// accesses, for example: -// -// package mypkg -// var ( -// configKey = viperutil.KeyPartial("foo.bar.mypkg") -// v1 = viperutil.NewValue(configKey("v1"), ...) // bound to "foo.bar.mypkg.v1" -// v2 = viperutil.NewValue(configKey("v2"), ...) // bound to "foo.bar.mypkg.v2" -// ... -// ) -func KeyPartial(prefix string) func(string) string { - return func(key string) string { - return Key(prefix, key) - } -} diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go deleted file mode 100644 index 6b99cca13f4..00000000000 --- a/go/viperutil/viper.go +++ /dev/null @@ -1,336 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -Package viperutil provides additional functions for Vitess's use of viper for -managing configuration of various components. - -Example usage, without dynamic reload: - - var ( - tracingServer = viperutil.NewValue( - "trace.service", - viper.GetString, - viperutil.WithFlags[string]("tracer"), - viperutil.Withdefault("noop"), - ) - ) - - func init() { - servenv.OnParse(func(fs *pflag.FlagSet) { - fs.String("tracer", tracingServer.Value(), "tracing service to use") - tracingServer.Bind(nil, fs) - }) - } - - func StartTracing(serviceName string) io.Closer { - backend := tracingServer.Get() - factory, ok := tracingBackendFactories[backend] - ... - } - -Example usage, with dynamic reload: - - var ( - syncedViper = vipersync.New() - syncedGetDuration = vipersync.AdaptGetter(syncedViper, func(v *viper.Viper) func(key string) time.Duration { - return v.GetDuration - }) - syncedGetInt = vipersync.AdaptGetter(syncedViper, func(v *viper.Viper) func(key string) int { - return v.GetInt - }) - - // Convenience to remove need to repeat this module's prefix constantly. - keyPrefix = viperutil.KeyPrefix("grpc.client") - - keepaliveTime = viperutil.NewValue( - keyPrefix("keepalive.time"), - syncedGetDuration, - viperutil.WithFlags[time.Duration]("grpc_keepalive_time"), - viperutil.WithDefault(10 * time.Second), - ) - keepaliveTimeout = viperutil.NewValue( - keyPrefix("keepalive.timeout"), - syncedGetDuration, - viperutil.WithFlags[time.Duration]("grpc_keepalive_timeout"), - viperutil.WithDefault(10 * time.Second), - ) - - initialConnWindowSize = viperutil.NewValue( - keyPrefix("initial_conn_window_size"), - syncedGetInt, - viperutil.WithFlags[int]("grpc_initial_conn_window_size"), - ) - ) - - func init() { - binaries := []string{...} - for _, cmd := range binaries { - servenv.OnParseFor(cmd, func(fs *pflag.FlagSet) { - - }) - } - - servenv.OnConfigLoad(func() { - // Watch the config file for changes to our synced variables. - syncedViper.Watch(viper.ConfigFileUsed()) - }) - } - - func DialContext(ctx context.Context, target string, failFast FailFast, opts ...grpc.DialOption) (*grpc.ClientConn, error) { - ... - - // Load our dynamic variables, thread-safely. - kaTime := keepaliveTime.Fetch() - kaTimeout := keepaliveTimeout.Fetch() - if kaTime != 0 || kaTimeout != 0 { - kp := keepalive.ClientParameters{ - Time: kaTime, - Timeout: kaTimeout, - PermitWithoutStream: true, - } - newopts = append(newopts, grpc.WithKeepaliveParams(kp)) - } - - if size := initialConnWindowSize.Fetch(); size != 0 { - newopts = append(newopts, grpc.WithInitialConnWindowSize(int32(size))) - } - - // and so on ... - - return grpc.DialContext(ctx, target, newopts...) - } -*/ -package viperutil - -import ( - "fmt" - - "github.com/spf13/pflag" - "github.com/spf13/viper" -) - -type Option[T any] func(val *Value[T]) - -func WithAliases[T any](aliases ...string) Option[T] { - return func(val *Value[T]) { - val.aliases = append(val.aliases, aliases...) - } -} - -func WithDefault[T any](t T) Option[T] { - return func(val *Value[T]) { - val.hasDefault, val.value = true, t - } -} - -func WithEnvVars[T any](envvars ...string) Option[T] { - return func(val *Value[T]) { - val.envvars = append(val.envvars, envvars...) - } -} - -func WithFlags[T any](flags ...string) Option[T] { - return func(val *Value[T]) { - val.flagNames = append(val.flagNames, flags...) - } -} - -type Value[T any] struct { - key string - value T - - aliases []string - flagNames []string - envvars []string - hasDefault bool - - resolve func(string) T - bound chan struct{} - loaded chan struct{} -} - -// NewValue returns a viper-backed value with the given key, lookup function, -// and bind options. -func NewValue[T any](key string, getFunc func(string) T, opts ...Option[T]) *Value[T] { - val := &Value[T]{ - key: key, - resolve: getFunc, - bound: make(chan struct{}, 1), - loaded: make(chan struct{}, 1), - } - - for _, opt := range opts { - opt(val) - } - - val.bound <- struct{}{} - val.loaded <- struct{}{} - - return val -} - -// Bind binds the value to flags, aliases, and envvars, depending on the Options -// the value was created with. -// -// If the passed-in viper is nil, the global viper instance is used. -// -// If the passed-in flag set is nil, flag binding is skipped. Otherwise, if any -// flag names do not match the value's canonical key, they are registered as -// additional aliases for the value's key. This function panics if a flag name -// is specified that is not defined on the flag set. -// -// Bind is not safe to call concurrently with other calls to Bind, Fetch, Get, -// or Value. It is recommended to call Bind at least once before calling those -// other 3 methods; otherwise the default or zero value for T will be returned. -func (val *Value[T]) Bind(v *viper.Viper, fs *pflag.FlagSet) { - select { - case _, ok := <-val.bound: - if ok { - defer func() { val.bound <- struct{}{} }() - } - } - - val.bind(v, fs) -} - -func (val *Value[T]) tryBind(v *viper.Viper, fs *pflag.FlagSet) { - select { - case _, ok := <-val.bound: - if ok { - val.bind(v, fs) - close(val.bound) - } - } -} - -func (val *Value[T]) bind(v *viper.Viper, fs *pflag.FlagSet) { - if v == nil { - v = viper.GetViper() - } - - aliases := val.aliases - - // Bind any flags, additionally aliasing flag names to the canonical key for - // this value. - if fs != nil { - for _, name := range val.flagNames { - f := fs.Lookup(name) - if f == nil { - // TODO: Don't love "configuration error" here as it might make - // users think _they_ made a config error, but really it is us, - // the Vitess authors, that have misconfigured a flag binding in - // the source code somewhere. - panic(fmt.Sprintf("configuration error: attempted to bind %s to unspecified flag %s", val.key, name)) - } - - if f.Name != val.key { - aliases = append(aliases, f.Name) - } - - // We are deliberately ignoring the error value here because it - // only occurs when `f == nil` and we check for that ourselves. - _ = v.BindPFlag(val.key, f) - } - } - - // Bind any aliases. - for _, alias := range aliases { - v.RegisterAlias(alias, val.key) - } - - // Bind any envvars. - if len(val.envvars) > 0 { - vars := append([]string{val.key}, val.envvars...) - // Again, ignoring the error value here, which is non-nil only when - // `len(vars) == 0`. - _ = v.BindEnv(vars...) - } - - // Set default value. - if val.hasDefault { - v.SetDefault(val.key, val.value) - } -} - -// Fetch returns the underlying value from the backing viper. -func (val *Value[T]) Fetch() T { - // Prior to fetching anything, bind the value to the global viper if it - // hasn't been bound to some viper already. - val.tryBind(nil, nil) - return val.resolve(val.key) -} - -// Get returns the underlying value, loading from the backing viper only on -// first use. For dynamic reloading, use Fetch. -func (val *Value[T]) Get() T { - val.tryBind(nil, nil) - select { - case _, ok := <-val.loaded: - if ok { - // We're the first to load; resolve the key and set the value. - val.value = val.resolve(val.key) - close(val.loaded) - } - } - - return val.value -} - -// Value returns the current underlying value. If a value has not been -// previously loaded (either via Fetch or Get), and the value was not created -// with a WithDefault option, the return value is whatever the zero value for -// the type T is. -func (val *Value[T]) Value() (value T) { - select { - case _, ok := <-val.loaded: - if ok { - // No one has loaded yet, and no one is loading. - // Return the value passed to us in the constructor, and - // put the sentinel value back so another Get call can proceed - // to actually load from the backing viper. - value = val.value - val.loaded <- struct{}{} - return value - } - } - - return val.value -} - -/* -old api - -func (v *Value[T]) BindFlag(f *pflag.Flag, aliases ...string) { - BindFlagWithAliases(f, v.key, aliases...) -} - -func (v *Value[T]) BindEnv(envvars ...string) { - viper.BindEnv(append([]string{v.key}, envvars...)...) -} - -func BindFlagWithAliases(f *pflag.Flag, canonicalKey string, aliases ...string) { - viper.BindPFlag(canonicalKey, f) - if canonicalKey != f.Name { - aliases = append(aliases, f.Name) - } - - for _, alias := range aliases { - viper.RegisterAlias(alias, canonicalKey) - } -} - -*/ diff --git a/go/viperutil/viperget/doc.go b/go/viperutil/viperget/doc.go deleted file mode 100644 index a99c45195ac..00000000000 --- a/go/viperutil/viperget/doc.go +++ /dev/null @@ -1,33 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreedto in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -package viperget provides custom getter functions for retrieving values from a -viper. - -For example, to retrieve a key as a topodatapb.TabletType variable, the -viperutil.Value should be instantiated like: - - // Passing `nil` to viperget.TabletType will have it lookup the key on the - // current global viper. - viperutil.NewValue[topodatapb.TabletType](myKey, viperget.TabletType(nil), opts...) - // If using a specific viper instance, instead do the following. - viperutil.NewValue[topodatapb.TabletType](myKey, viperget.TabletType(myViper), opts...) - -This is a subpackage instead of being part of viperutil directly in order to -avoid an import cycle between go/vt/log and viperutil. -*/ -package viperget diff --git a/go/viperutil/viperget/duration_or_int.go b/go/viperutil/viperget/duration_or_int.go deleted file mode 100644 index 463048df0a7..00000000000 --- a/go/viperutil/viperget/duration_or_int.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreedto in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package viperget - -import ( - "time" - - "github.com/spf13/viper" -) - -func DurationOrInt(v *viper.Viper, intFallbackUnit time.Duration) func(key string) time.Duration { - if v == nil { - v = viper.GetViper() - } - - return func(key string) time.Duration { - if !v.IsSet(key) { - return time.Duration(0) - } - - if d := v.GetDuration(key); d != 0 { - return d - } - - return time.Duration(v.GetInt(key)) * intFallbackUnit - } -} diff --git a/go/viperutil/viperget/path.go b/go/viperutil/viperget/path.go deleted file mode 100644 index 080de7de0d4..00000000000 --- a/go/viperutil/viperget/path.go +++ /dev/null @@ -1,21 +0,0 @@ -package viperget - -import ( - "strings" - - "github.com/spf13/viper" -) - -func Path(v *viper.Viper) func(key string) []string { - return func(key string) (paths []string) { - for _, val := range v.GetStringSlice(key) { - if val != "" { - for _, path := range strings.Split(val, ":") { - paths = append(paths, path) - } - } - } - - return paths - } -} diff --git a/go/viperutil/viperget/tablet_type.go b/go/viperutil/viperget/tablet_type.go deleted file mode 100644 index 71088d3f357..00000000000 --- a/go/viperutil/viperget/tablet_type.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package viperget - -import ( - "github.com/spf13/viper" - - "vitess.io/vitess/go/vt/log" - topodatapb "vitess.io/vitess/go/vt/proto/topodata" - "vitess.io/vitess/go/vt/topo/topoproto" -) - -func TabletType(v *viper.Viper) func(key string) topodatapb.TabletType { - if v == nil { - v = viper.GetViper() - } - - return func(key string) (ttype topodatapb.TabletType) { - if !v.IsSet(key) { - return - } - - switch val := v.Get(key).(type) { - case int: - ttype = topodatapb.TabletType(int32(val)) - case string: - var err error - if ttype, err = topoproto.ParseTabletType(val); err != nil { - // TODO: decide how we want to handle these cases. - log.Warningf("GetTabletTypeValue failed to parse tablet type: %s. Defaulting to TabletType %s.\n", err.Error(), topoproto.TabletTypeLString(ttype)) - } - default: - // TODO: decide how we want to handle this case. - log.Warningf("GetTabletTypeValue: invalid Go type %T for TabletType; must be string or int. Defaulting to TabletType %s.\n", val, ttype) - } - - return - } -} diff --git a/go/viperutil/vipersync/viper.go b/go/viperutil/vipersync/viper.go deleted file mode 100644 index c13b47eb809..00000000000 --- a/go/viperutil/vipersync/viper.go +++ /dev/null @@ -1,142 +0,0 @@ -package vipersync - -import ( - "errors" - "fmt" - "sync" - - "github.com/fsnotify/fsnotify" - "github.com/spf13/pflag" - "github.com/spf13/viper" - - "vitess.io/vitess/go/viperutil" -) - -// TODO: document all this -// TODO: use jww for logging within go/viperutil - -type Viper struct { - disk *viper.Viper - live *viper.Viper - keys map[string]lockable - - subscribers []chan<- struct{} - watchingConfig bool -} - -// lockable represents a structure that contains a lockable field. -// -// we need this interface in order to store our map of synced keys in the synced -// viper. unfortunately, we cannot do -// -// for _, key := range sv.keys { -// key.m.Lock() -// defer key.m.Unlock() -// } -// -// because generics don't (yet?) allow a collection of _different_ generic types -// (e.g. you can't have []T{1, "hello", false}, because they are not all the same T), -// so we abstract over the interface. it's not exported because no one outside -// this package should be accessing a synced key's mutex. -type lockable interface { - locker() sync.Locker -} - -type syncedKey[T any] struct { - m sync.RWMutex - get func(key string) T -} - -func (sk *syncedKey[T]) locker() sync.Locker { - return &sk.m -} - -func New() *Viper { - return &Viper{ - disk: viper.New(), - live: viper.New(), - keys: map[string]lockable{}, - } -} - -func (v *Viper) Watch(cfg string) error { - if v.watchingConfig { - // TODO: declare an exported error and include the filename in wrapped - // error. - return errors.New("duplicate watch") - } - - if err := viperutil.ReadInConfig(v.disk, viperutil.ConfigOptions{ - File: cfg, - }, viperutil.ErrorOnConfigFileNotFound); err != nil { - return err // TODO: wrap error - } - - v.watchingConfig = true - v.loadFromDisk() - - v.disk.OnConfigChange(func(in fsnotify.Event) { - // Inform each key that an update is coming. - for _, key := range v.keys { - key.locker().Lock() - // This won't fire until after the config has been updated on v.live. - defer key.locker().Unlock() - } - - // Now, every key is blocked from reading; we can atomically swap the - // config on disk for the config in memory. - v.loadFromDisk() - - for _, ch := range v.subscribers { - select { - case ch <- struct{}{}: - default: - } - } - }) - v.disk.WatchConfig() - - return nil -} - -func (v *Viper) loadFromDisk() { - // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an - // older version of viper it used to actually handle errors, but now it - // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. - _ = v.live.MergeConfigMap(v.disk.AllSettings()) -} - -func (v *Viper) Notify(ch chan<- struct{}) { - if v.watchingConfig { - panic("cannot Notify after starting to watch config") - } - - v.subscribers = append(v.subscribers, ch) -} - -func BindValue[T any](v *Viper, value *viperutil.Value[T], fs *pflag.FlagSet) { - value.Bind(v.live, fs) -} - -func AdaptGetter[T any](v *Viper, key string, getter func(v *viper.Viper) func(key string) T) func(key string) T { - if v.watchingConfig { - panic("cannot adapt getter to synchronized viper which is already watching a config") - } - - if _, ok := v.keys[key]; ok { - panic(fmt.Sprintf("already adapted a getter for key %s", key)) - } - - sk := &syncedKey[T]{ - get: getter(v.live), - } - - v.keys[key] = sk - - return func(key string) T { - sk.m.RLock() - defer sk.m.RUnlock() - - return sk.get(key) - } -} diff --git a/go/viperutil/vipersync/viper_test.go b/go/viperutil/vipersync/viper_test.go deleted file mode 100644 index e82bd5d08f9..00000000000 --- a/go/viperutil/vipersync/viper_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package vipersync_test - -import ( - "context" - "encoding/json" - "math/rand" - "os" - "sync" - "testing" - "time" - - "github.com/fsnotify/fsnotify" - "github.com/spf13/viper" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/viperutil" - "vitess.io/vitess/go/viperutil/vipersync" -) - -func TestWatchConfig(t *testing.T) { - type config struct { - A, B int - } - - tmp, err := os.CreateTemp(".", "TestWatchConfig_*.json") - require.NoError(t, err) - t.Cleanup(func() { os.Remove(tmp.Name()) }) - - stat, err := os.Stat(tmp.Name()) - require.NoError(t, err) - - writeConfig := func(a, b int) error { - data, err := json.Marshal(&config{A: a, B: b}) - if err != nil { - return err - } - - return os.WriteFile(tmp.Name(), data, stat.Mode()) - } - writeRandomConfig := func() error { - a, b := rand.Intn(100), rand.Intn(100) - return writeConfig(a, b) - } - - require.NoError(t, writeRandomConfig()) - - v := viper.New() - v.SetConfigFile(tmp.Name()) - require.NoError(t, v.ReadInConfig()) - - wCh, rCh := make(chan struct{}), make(chan struct{}) - v.OnConfigChange(func(in fsnotify.Event) { - select { - case <-rCh: - return - default: - } - - wCh <- struct{}{} - // block forever to prevent this viper instance from double-updating. - <-rCh - }) - v.WatchConfig() - - // Make sure that basic, unsynchronized WatchConfig is set up before - // beginning the actual test. - a, b := v.GetInt("a"), v.GetInt("b") - require.NoError(t, writeConfig(a+1, b+1)) - <-wCh // wait for the update to finish - - time.Sleep(time.Millisecond * 1000) - require.Equal(t, a+1, v.GetInt("a")) - require.Equal(t, b+1, v.GetInt("b")) - - rCh <- struct{}{} - - // Now, set up our synchronized viper and do a bunch of concurrent reads/writes. - v = viper.New() - - sv := vipersync.New() - A := viperutil.NewValue("a", - vipersync.AdaptGetter(sv, "a", func(v *viper.Viper) func(key string) int { - return v.GetInt - }), - ) - B := viperutil.NewValue("b", - vipersync.AdaptGetter(sv, "b", func(v *viper.Viper) func(key string) int { - return v.GetInt - }), - ) - vipersync.BindValue(sv, A, nil) - vipersync.BindValue(sv, B, nil) - - require.NoError(t, sv.Watch(tmp.Name())) - - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(context.Background()) - - // Sleep between 25 and 50ms between reads. - readJitter := func() time.Duration { - return time.Duration(jitter(25, 50)) * time.Millisecond - } - // Sleep between 75 and 125ms between writes. - writeJitter := func() time.Duration { - return time.Duration(jitter(75, 125)) * time.Millisecond - } - - for i := 0; i < 2; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - - for { - select { - case <-ctx.Done(): - return - default: - } - - switch i % 2 { - case 0: - A.Fetch() - case 1: - B.Fetch() - } - - time.Sleep(readJitter()) - } - }(i) - } - - for i := 0; i < 100; i++ { - require.NoError(t, writeRandomConfig()) - time.Sleep(writeJitter()) - } - - cancel() - wg.Wait() -} - -func jitter(min, max int) int { - return min + rand.Intn(max-min+1) -} From 43b114be33eb145308fae5232e9bcaf42c1daaea Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 16 Nov 2022 14:43:50 -0500 Subject: [PATCH 051/100] mv go/viperutil/v2 => go/viperutil and update everythingggggggggg Signed-off-by: Andrew Mason --- go/trace/plugin_datadog.go | 2 +- go/trace/plugin_jaeger.go | 2 +- go/trace/trace.go | 2 +- go/trace/trace_test.go | 2 +- go/viperutil/{v2 => }/config.go | 8 ++++---- go/viperutil/{v2 => }/config_test.go | 0 go/viperutil/{v2 => }/debug/debug.go | 2 +- go/viperutil/{v2 => }/errors.go | 4 ++-- go/viperutil/{v2 => }/funcs/decode.go | 0 go/viperutil/{v2 => }/funcs/get.go | 0 go/viperutil/{v2 => }/funcs/get_test.go | 6 +++--- go/viperutil/{v2 => }/get_func.go | 0 go/viperutil/{v2 => }/get_func_test.go | 0 go/viperutil/{v2 => }/internal/log/log.go | 0 go/viperutil/{v2 => }/internal/registry/registry.go | 2 +- go/viperutil/{v2 => }/internal/sync/sync.go | 0 go/viperutil/{v2 => }/internal/sync/sync_test.go | 6 +++--- go/viperutil/{v2 => }/internal/value/value.go | 4 ++-- go/viperutil/{v2 => }/value.go | 2 +- go/viperutil/{v2 => }/viper.go | 6 +++--- go/viperutil/{v2 => }/vipertest/stub.go | 4 ++-- go/vt/mysqlctl/azblobbackupstorage/azblob.go | 2 +- go/vt/servenv/servenv.go | 2 +- go/vt/topo/locks.go | 2 +- 24 files changed, 29 insertions(+), 29 deletions(-) rename go/viperutil/{v2 => }/config.go (97%) rename go/viperutil/{v2 => }/config_test.go (100%) rename go/viperutil/{v2 => }/debug/debug.go (93%) rename go/viperutil/{v2 => }/errors.go (91%) rename go/viperutil/{v2 => }/funcs/decode.go (100%) rename go/viperutil/{v2 => }/funcs/get.go (100%) rename go/viperutil/{v2 => }/funcs/get_test.go (90%) rename go/viperutil/{v2 => }/get_func.go (100%) rename go/viperutil/{v2 => }/get_func_test.go (100%) rename go/viperutil/{v2 => }/internal/log/log.go (100%) rename go/viperutil/{v2 => }/internal/registry/registry.go (97%) rename go/viperutil/{v2 => }/internal/sync/sync.go (100%) rename go/viperutil/{v2 => }/internal/sync/sync_test.go (95%) rename go/viperutil/{v2 => }/internal/value/value.go (97%) rename go/viperutil/{v2 => }/value.go (97%) rename go/viperutil/{v2 => }/viper.go (97%) rename go/viperutil/{v2 => }/vipertest/stub.go (94%) diff --git a/go/trace/plugin_datadog.go b/go/trace/plugin_datadog.go index 7480b9d85a9..2755742f9e0 100644 --- a/go/trace/plugin_datadog.go +++ b/go/trace/plugin_datadog.go @@ -9,7 +9,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer" ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" ) var ( diff --git a/go/trace/plugin_jaeger.go b/go/trace/plugin_jaeger.go index 1341d637481..9da9546c2ff 100644 --- a/go/trace/plugin_jaeger.go +++ b/go/trace/plugin_jaeger.go @@ -23,7 +23,7 @@ import ( "github.com/spf13/pflag" "github.com/uber/jaeger-client-go/config" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" ) diff --git a/go/trace/trace.go b/go/trace/trace.go index d2d1d126d96..7c43b4afedc 100644 --- a/go/trace/trace.go +++ b/go/trace/trace.go @@ -28,7 +28,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" ) diff --git a/go/trace/trace_test.go b/go/trace/trace_test.go index 5605af18d3f..08027a35c85 100644 --- a/go/trace/trace_test.go +++ b/go/trace/trace_test.go @@ -26,7 +26,7 @@ import ( "github.com/spf13/viper" "google.golang.org/grpc" - "vitess.io/vitess/go/viperutil/v2/vipertest" + "vitess.io/vitess/go/viperutil/vipertest" ) func TestFakeSpan(t *testing.T) { diff --git a/go/viperutil/v2/config.go b/go/viperutil/config.go similarity index 97% rename from go/viperutil/v2/config.go rename to go/viperutil/config.go index ad59687a8a9..69482eb569c 100644 --- a/go/viperutil/v2/config.go +++ b/go/viperutil/config.go @@ -27,10 +27,10 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/v2/funcs" - "vitess.io/vitess/go/viperutil/v2/internal/log" - "vitess.io/vitess/go/viperutil/v2/internal/registry" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil/funcs" + "vitess.io/vitess/go/viperutil/internal/log" + "vitess.io/vitess/go/viperutil/internal/registry" + "vitess.io/vitess/go/viperutil/internal/value" ) var ( diff --git a/go/viperutil/v2/config_test.go b/go/viperutil/config_test.go similarity index 100% rename from go/viperutil/v2/config_test.go rename to go/viperutil/config_test.go diff --git a/go/viperutil/v2/debug/debug.go b/go/viperutil/debug/debug.go similarity index 93% rename from go/viperutil/v2/debug/debug.go rename to go/viperutil/debug/debug.go index fb918b7e95b..ad045a8fa2f 100644 --- a/go/viperutil/v2/debug/debug.go +++ b/go/viperutil/debug/debug.go @@ -19,7 +19,7 @@ package debug import ( "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/v2/internal/registry" + "vitess.io/vitess/go/viperutil/internal/registry" ) func Debug() { diff --git a/go/viperutil/v2/errors.go b/go/viperutil/errors.go similarity index 91% rename from go/viperutil/v2/errors.go rename to go/viperutil/errors.go index eddeb699332..bb2018b4e88 100644 --- a/go/viperutil/v2/errors.go +++ b/go/viperutil/errors.go @@ -17,8 +17,8 @@ limitations under the License. package viperutil import ( - "vitess.io/vitess/go/viperutil/v2/internal/sync" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil/internal/sync" + "vitess.io/vitess/go/viperutil/internal/value" ) var ( diff --git a/go/viperutil/v2/funcs/decode.go b/go/viperutil/funcs/decode.go similarity index 100% rename from go/viperutil/v2/funcs/decode.go rename to go/viperutil/funcs/decode.go diff --git a/go/viperutil/v2/funcs/get.go b/go/viperutil/funcs/get.go similarity index 100% rename from go/viperutil/v2/funcs/get.go rename to go/viperutil/funcs/get.go diff --git a/go/viperutil/v2/funcs/get_test.go b/go/viperutil/funcs/get_test.go similarity index 90% rename from go/viperutil/v2/funcs/get_test.go rename to go/viperutil/funcs/get_test.go index b6008fa062f..f20cc051a22 100644 --- a/go/viperutil/v2/funcs/get_test.go +++ b/go/viperutil/funcs/get_test.go @@ -21,9 +21,9 @@ import ( "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/v2" - "vitess.io/vitess/go/viperutil/v2/funcs" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/funcs" + "vitess.io/vitess/go/viperutil/internal/value" ) func ExampleGetPath() { diff --git a/go/viperutil/v2/get_func.go b/go/viperutil/get_func.go similarity index 100% rename from go/viperutil/v2/get_func.go rename to go/viperutil/get_func.go diff --git a/go/viperutil/v2/get_func_test.go b/go/viperutil/get_func_test.go similarity index 100% rename from go/viperutil/v2/get_func_test.go rename to go/viperutil/get_func_test.go diff --git a/go/viperutil/v2/internal/log/log.go b/go/viperutil/internal/log/log.go similarity index 100% rename from go/viperutil/v2/internal/log/log.go rename to go/viperutil/internal/log/log.go diff --git a/go/viperutil/v2/internal/registry/registry.go b/go/viperutil/internal/registry/registry.go similarity index 97% rename from go/viperutil/v2/internal/registry/registry.go rename to go/viperutil/internal/registry/registry.go index afe1dd7b30b..c8b03ad4e54 100644 --- a/go/viperutil/v2/internal/registry/registry.go +++ b/go/viperutil/internal/registry/registry.go @@ -20,7 +20,7 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/v2/internal/sync" + "vitess.io/vitess/go/viperutil/internal/sync" ) var ( diff --git a/go/viperutil/v2/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go similarity index 100% rename from go/viperutil/v2/internal/sync/sync.go rename to go/viperutil/internal/sync/sync.go diff --git a/go/viperutil/v2/internal/sync/sync_test.go b/go/viperutil/internal/sync/sync_test.go similarity index 95% rename from go/viperutil/v2/internal/sync/sync_test.go rename to go/viperutil/internal/sync/sync_test.go index 32fef4dc4f1..d97f8b3d6da 100644 --- a/go/viperutil/v2/internal/sync/sync_test.go +++ b/go/viperutil/internal/sync/sync_test.go @@ -29,9 +29,9 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/viperutil/v2" - vipersync "vitess.io/vitess/go/viperutil/v2/internal/sync" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil" + vipersync "vitess.io/vitess/go/viperutil/internal/sync" + "vitess.io/vitess/go/viperutil/internal/value" ) func TestWatchConfig(t *testing.T) { diff --git a/go/viperutil/v2/internal/value/value.go b/go/viperutil/internal/value/value.go similarity index 97% rename from go/viperutil/v2/internal/value/value.go rename to go/viperutil/internal/value/value.go index a540668c346..344f88ed967 100644 --- a/go/viperutil/v2/internal/value/value.go +++ b/go/viperutil/internal/value/value.go @@ -23,8 +23,8 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/v2/internal/registry" - "vitess.io/vitess/go/viperutil/v2/internal/sync" + "vitess.io/vitess/go/viperutil/internal/registry" + "vitess.io/vitess/go/viperutil/internal/sync" ) // Registerable is the subset of the interface exposed by Values (which is diff --git a/go/viperutil/v2/value.go b/go/viperutil/value.go similarity index 97% rename from go/viperutil/v2/value.go rename to go/viperutil/value.go index 1ca88b0737a..3fc8b6b6a7d 100644 --- a/go/viperutil/v2/value.go +++ b/go/viperutil/value.go @@ -19,7 +19,7 @@ package viperutil import ( "github.com/spf13/pflag" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil/internal/value" ) var ( diff --git a/go/viperutil/v2/viper.go b/go/viperutil/viper.go similarity index 97% rename from go/viperutil/v2/viper.go rename to go/viperutil/viper.go index c45e8f323d0..12ef4ca53f8 100644 --- a/go/viperutil/v2/viper.go +++ b/go/viperutil/viper.go @@ -23,7 +23,7 @@ variables that are the result of calling Configure, for example in package trace package trace - import "vitess.io/vitess/go/viperutil/v2" + import "vitess.io/vitess/go/viperutil" var ( modulePrefix = viperutil.KeyPrefixFunc("trace") @@ -51,7 +51,7 @@ to those flags, as appropriate: import ( "github.com/spf13/pflag" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/servenv" ) @@ -80,7 +80,7 @@ import ( "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil/internal/value" ) // Options represents the various options used to control how Values are diff --git a/go/viperutil/v2/vipertest/stub.go b/go/viperutil/vipertest/stub.go similarity index 94% rename from go/viperutil/v2/vipertest/stub.go rename to go/viperutil/vipertest/stub.go index fe5cd945fde..700d3639566 100644 --- a/go/viperutil/v2/vipertest/stub.go +++ b/go/viperutil/vipertest/stub.go @@ -22,8 +22,8 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/assert" - "vitess.io/vitess/go/viperutil/v2" - "vitess.io/vitess/go/viperutil/v2/internal/value" + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/internal/value" ) // Stub stubs out a given value to use the passed-in viper to retrieve its diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 0b1c2eb33fe..2f382bdbfc6 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -32,7 +32,7 @@ import ( "github.com/Azure/azure-storage-blob-go/azblob" "github.com/spf13/pflag" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index c2af311ce33..c073c199a97 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -44,7 +44,7 @@ import ( "vitess.io/vitess/go/netutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/grpccommon" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" diff --git a/go/vt/topo/locks.go b/go/vt/topo/locks.go index 7e935ab7792..e40792fcb9c 100644 --- a/go/vt/topo/locks.go +++ b/go/vt/topo/locks.go @@ -29,7 +29,7 @@ import ( _flag "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/viperutil/v2" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/servenv" From 4d5151ffee50ef781a0525248a7849ed58c121c2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 18 Nov 2022 14:58:05 -0500 Subject: [PATCH 052/100] cleanup todo that is done Signed-off-by: Andrew Mason --- go/viperutil/viper.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index 12ef4ca53f8..0106b8e95e8 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -70,7 +70,6 @@ Finally, after a call to `viperutil.LoadConfig` (which is done as a part of For more details, refer to the package documentation, as well as the documents in doc/viper/. -- TODO: remove original API and `git mv` v2 up a level, adjusting imports - TODO: markdown doc under doc/ tree giving an overview. */ package viperutil From 1d5762536fc804e6a15da5692c08daeb4214e633 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 20 Nov 2022 18:48:45 -0500 Subject: [PATCH 053/100] start documenting new stuff Signed-off-by: Andrew Mason --- doc/viper/viper.md | 146 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index 422986020ed..bbb211c30db 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -14,6 +14,150 @@ It acts as a registry for configuration values coming from a variety of sources, It is used by a wide variety of Go projects, including [hugo][hugo] and [the kubernetes operator][kops]. +## "Normal" Usage + +Normally, and if you were to follow the examples on the viper documentation, you "just" load in a config file, maybe bind some flags, and then load values all across your codebase, like so: + +```go +// cmd/main.go +package main + +import ( + "log" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "example.com/pkg/stuff" +) + +func main() { + pflag.String("name", "", "name to print") + pflag.Parse() + + viper.AddConfigPath(".") + viper.AddConfigPath("/var/mypkg") + + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + log.Fatal(err) + } + } + + viper.BindPFlags(pflag.CommandLine) + viper.BindEnv("name", "MY_COOL_ENVVAR") + + stuff.Do() +} + +// pkg/stuff/do_stuff.go +package stuff + +import ( + "fmt" + + "github.com/spf13/viper" +) + +func Do() { + fmt.Println(viper.GetString("name")) +} +``` + +While this example is great for getting started with `viper` quickly — it is very easy, and very fast, to _write_ go from nothing to working code — it's not likely to scale well for a codebase the size and complexity of Vitess, for several reasons: + +1. Everything is globally-accessible. + + Currently, most of the config values in Vitess modules are un-exported (and we un-exported more of these during the `pflag` migration). + This is a good thing, as it gives each module control over how its configuration values are used, rather than allowing the raw values to leak across package boundaries. + +1. [Magical][a_theory_of_modern_go] access and lack of compile-time safety. + + In the above example, `package stuff` just "happens to know" that (1) `package main` binds a value to `viper` with the key `"name"` and (2) `"name"` is going to be bound to a `string` value specifically. + + If `package main` ever changes either of these two facts, `package stuff` is going to break (precisely how it breaks depends on what the changes are), and there's no way to catch this _before_ runtime without writing additional linters. (Note this is strongly-related to point 1 above). + +1. Hard to document. + + `viper` does not provide any sort of automatic documentation-generation code (we'll discuss this more later), so we will need to write our own tooling if we want to update our flag documentation with information like "this flag is also settable via this config key and these environment variables". + + If anyone anywhere can just magically try to read a value from the global registry without first declaring (1) that a config value with that key _should_ exist; (2) what flags, aliases, environment variables it reads from; and (3) what type it is, then writing that tooling is going to be vastly more complicated, and possibly impossible to do correctly. + +So, we use an approach that requires a bit more verbosity up-front to mitigate these drawbacks to the simpler approach. + +## Our Approach + +Instead of relying on the global `viper.Viper` singleton, we use a shim layer introduced in `package viperutil` to configure values in a standardized way across the entire Vitess codebase. + +This function, `Configure`, then returns a value object, with a `Get` method that returns the actual value from the viper registry. +Packages may then choose to export their config values, or not, as they see fit for their API consumers. + +### `Configure` Options + +In order to properly configure a value for use, `Configure` needs to know, broadly speaking, three things: + +1. The key name being bound. +1. What "things" it should be bound to (i.e. other keys via aliases, environment variables, and flag names), as well as if it has a default value. +1. How to `Get` it out of a viper. + +`Configure`, therefore, has the following signature: + +```go +func Configure[T any](key string, options Options[T]) Value[T] +``` + +The first parameter provides the key name (point 1 of our above list); all other information is provided via various `Options` fields, which looks like: + +```go +type Options[T any] struct { + // what "things" to bind to + Aliases []string + FlagName string + EnvVars []string + + // default, if any + Default T + + // whether it can reload or not (more on this later) + Dynamic bool + + // how to "get" it from a viper (more on this slightly less later) + GetFunc func(v *viper.Viper) func(key string) T +} +``` + +### `Get` funcs + +In most cases, module authors will not need to specify a `GetFunc` option, since, if not provided, `viperutil` will do its best to provide a sensible default for the given type `T`. + +This requires a fair amount of `reflect`ion code, which we won't go into here, and unfortunately cannot support even all primitive types (notably, array (not slice!!) types). +In these cases, the `GetFuncForType` will panic, allowing the module author to catch this during testing of their package. +They may then provide their own `GetFunc`. + +Authors may also want to provide their own `GetFunc` to provide additional logic to load a value even for types supported by `GetFuncForType` (for example, post-processing a string to ensure it is always lowercase). + +The full suite of types, both supported and panic-inducing, are documented by way of unit tests in [`go/viperutil/get_func_test.go`](../../go/viperutil/get_func_test.go). + +### Dynamic values + +Values can be configured to be either static or dynamic. +Static values are loaded once at startup (more precisely, when `viperutil.LoadConfig` is called), and whatever value is loaded at the point will be the result of calling `Get` on that value for the remainder of the processes lifetime. +Dynamic values, conversely, may respond to config changes. + +In order for dynamic configs to be truly dynamic, `LoadConfig` must have found a config file (as opposed to pulling values entirely from defaults, flags, and environment variables). +If this is the case, a second viper shim, which backs the dynamic registry, will start a watch on that file, and any changes to that file will be reflected in the `Get` methods of any values configured with `Dynamic: true`. + +**An important caveat** is that viper on its own is not threadsafe, meaning that if a config reload is being processed while a value is being accessed, a race condition can occur. +To protect against this, the dynamic registry uses a second shim, [`sync.Viper`](../../go/viperutil/internal/sync/sync.go). +This works by assigning each dynamic value its own `sync.RWMutex`, and locking it for writes whenever a config change is detected. Value `GetFunc`s are then adapted to wrap the underlying get in a `m.RLock(); defer m.RUnlock()` layer. +This means that there's a potential throughput impact of using dynamic values, which module authors should be aware of when deciding to make a given value dynamic. + +### A brief aside on flags + +## Auto-Documentation + + + ## Common Usage Broadly speaking, there's two approaches for Vitess to using viper. @@ -187,3 +331,5 @@ TODO: see `go/viperutil/vipersync` [hugo]: https://github.com/gohugoio/hugo [kops]: https://github.com/kubernetes/kops + +[a_theory_of_modern_go]: https://peter.bourgon.org/blog/2017/06/09/theory-of-modern-go.html From 1df178c9d76f55a7d124446903fa05855f2cdda9 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 20 Nov 2022 18:50:12 -0500 Subject: [PATCH 054/100] delete most of the old docs Signed-off-by: Andrew Mason --- doc/viper/viper.md | 131 --------------------------------------------- 1 file changed, 131 deletions(-) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index bbb211c30db..8bec22cd012 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -158,123 +158,6 @@ This means that there's a potential throughput impact of using dynamic values, w -## Common Usage - -Broadly speaking, there's two approaches for Vitess to using viper. - -### Approach 1: Everything in the global registry. - -In this approach, we simply: - -```go -import ( - "github.com/spf13/pflag" - "github.com/spf13/viper" - - // See the section on viperutil for details on this package. - "vitess.io/vitess/go/viperutil" - "vitess.io/vitess/go/vt/servenv" -) - -var myValue = viperutil.NewValue( - "mykey", viper.GetString, - viperutil.WithFlags[string]("myflagname"), - viperutil.WithDefault("defaultvalue"), -) - -func init() { - servenv.OnParseFor("mycmd", func(fs *pflag.FlagSet) { - fs.String("myflagname", myValue.Value(), "help text for myflagname") - myValue.Bind( - /* nil here means use the global viper */ nil, - fs, - ) - }) -} - -func DoAThingWithMyValue() { - fmt.Println("mykey=", myValue.Get()) -} -``` - -Pros: -- Easy to read and write. -- Easy to debug (we can provide a flag to all binaries that results in calling `viper.Debug()`, which would dump out every setting from every vitess module). - -Cons: -- Requires us to be disciplined about cross-module boundaries. - - Anyone anywhere can then do `viper.GetString("mykey")` and retrieve the value. - - Even more scarily, anyone anywhere can _override_ the value via `viper.Set("mykey", 2)` (notice I've even changed the type here). - - Even more _more_ scarily, see below about threadsafety for how dangerous this can be. - -### Approach 2: Package-local vipers - -Instead of putting everything in the global registry, each package can declare a local `viper` instance and put its configuration there. Instead of the above example, this would look like: - -```go -import ( - "github.com/spf13/pflag" - "github.com/spf13/viper" - - "vitess.io/vitess/go/viperutil" - "vitess.io/vitess/go/vt/servenv" -) - -var ( - v = viper.New() - myValue = viperutil.NewValue( - "mykey", v.GetString, - viperutil.WithFlags[string]("myflagname"), - viperutil.WithDefault("defaultvalue"), - ), -) - -func init() { - servenv.OnParseFor("mycmd", func(fs *pflag.FlagSet) { - fs.String("myflagname", myValue.Value(), "help text for myflagname") - myValue.Bind( - /* bind to our package-local viper instance */ v, - fs, - ) - }) -} - -func DoAThingWithMyValue() { - fmt.Println("mykey=", myValue.Get()) -} -``` - -Pros: -- Maintains package-private nature of configuration values we currently have. - -Cons: -- No easy "show me all the debug settings everywhere". We would have to have each package expose a function to call, or come up with some other solution. -- Hard to read and write. - - "Wait, what is this `v` thing and where did it come from? - -### Approach 2.2: Readability changes via import alias. - -To address the readability issue in Approach 2, we could alias the viper import to let us write "normal" looking viper code: - -```go -import ( - _viper "github.com/spf13/viper" - - "vitess.io/vitess/go/viperutil" -) - -var ( - viper = _viper.New() - myValue = viperutil.NewValue( - "mykey", viper.GetString, - viperutil.WithFlags[string]("myflagname"), - viperutil.WithDefault("defaultvalue"), - ), -) -``` - -The problem _here_ (in addition to it being admittedly a little sneaky), is that if you were to open a new file in the same package, your IDE would very likely pick up the "viper" string and simply import the package without the alias (which also has a `GetString` function), and now you have two files in the same package, one working with the local variable, and the other working with the global registry, and it would be _verrrrry_ tricky to notice in code review. To address that we could write a simple linter to verify that (with the exception of explicitly allow-listed modules), all Vitess modules only ever import `viper` as the aliased version. - ## Config File(s) All vitess components will support taking a single, static (i.e. not "watched" via `WatchConfig`) configuration file. @@ -282,9 +165,6 @@ All vitess components will support taking a single, static (i.e. not "watched" v The file will be loaded via `ReadInConfig`. We will log a warning if no file was found, but will not abort (see [docs][viper_read_in_config_docs]). -Subsystems may allow for specifying additional (optionally dynamic, per their discretion, but see Caveats below) config files loaded separately from the main config. -They may choose to follow the global example of not aborting on `viper.ConfigFileNotFoundError` or not, per their discretion (**provided deviation from the norm is captured in that subsystems config flag usage**). - Flags for all binaries: - `--config-path` - Default: `$(pwd)` @@ -308,20 +188,9 @@ Flags for all binaries: - FlagType: `string` - Behavior: Instructs `ReadInConfig` to search in `ConfigPaths` for explicitly a file with this name. Takes precedence over `ConfigName`. -TODO: if we go with Approach 2.1 or Approach 2.2 in the above section, we need to work out a way to propagate the `ReadInConfig` outlined here from the global viper back to each of the package-local vipers. - -### Watching Configs - -TODO: see `go/viperutil/vipersync` - -## `go/viperutil` - -## `go/viperutil/viperget` - ## Caveats and Gotchas - [ ] case-(in)sensitivity. -- [ ] Threadsafety. - [ ] `Sub` is split-brain - [ ] `Unmarshal*` functions rely on `mapstructure` tags, not `json|yaml|...` tags. - [ ] Any config files/paths added _after_ calling `WatchConfig` will not get picked up. From d5ff18acefb713d3ff94bf47d7e21f15212dcb69 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 21 Nov 2022 07:50:30 -0500 Subject: [PATCH 055/100] flaggy flag docs Signed-off-by: Andrew Mason --- doc/viper/viper.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index 8bec22cd012..a4a5d962c71 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -154,6 +154,58 @@ This means that there's a potential throughput impact of using dynamic values, w ### A brief aside on flags +In the name of "we will catch as many mistakes as possible in tests" ("mistakes" here referring to typos in flag names, deleting a flag in one place but forgetting to clean up another reference, and so on), `Values` will panic at bind-time if they are configured to bind to a flag name that does not exist. +Then, **as long as every binary is at least invoked** (including just `mycmd --help`) in an end-to-end test, our CI will fail if we ever misconfigure a value in this way. + +However, since `Configure` handles the binding of defaults, aliases, and envirnomnent variables, and is usually called in `var` blocks, this binding can actually happen before the module registers its flags via the `servenv.{OnParse,OnParseFor}` hooks. +If we were to also bind any named flags at the point of `Configure`, this would cause panics even if the module later registered a flag with that name. +Therefore, we introduce a separate function, namely `viperutil.BindFlags`, which binds the flags on one or more values, which modules can call _after_ registering their flags, usually in the same `OnParse` hook function. +For example: + +```go +package azblobbackupstorage + +import ( + "github.com/spf13/pflag" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/vt/servenv" +) + +var ( + configKey = viperutil.KeyPrefixFunc("backup.storage.azblob") + + accountName = viperutil.Configure( + configKey("account.name"), + viperutil.Options[string]{ + EnvVars: []string{"VT_AZBLOB_ACCOUNT_NAME"}, + FlagName: "azblob_backup_account_name", + }, + ) + + accountKeyFile = viperutil.Configure( + configKey("account.key_file"), + viperutil.Options[string]{ + FlagName: "azblob_backup_account_key_file", + }, + ) +) + +func registerFlags(fs *pflag.FlagSet) { + fs.String("azblob_backup_account_name", accountName.Default(), "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") + fs.String("azblob_backup_account_key_file", accountKeyFile.Default(), "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") + + viperutil.BindFlags(fs, accountName, accountKeyFile) +} + +func init() { + servenv.OnParseFor("vtbackup", registerFlags) + servenv.OnParseFor("vtctl", registerFlags) + servenv.OnParseFor("vtctld", registerFlags) + servenv.OnParseFor("vttablet", registerFlags) +} +``` + ## Auto-Documentation From 2410b5de497709c8686db7f1539a7c2ece06fdf4 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 21 Nov 2022 13:54:45 -0500 Subject: [PATCH 056/100] config file stuff Signed-off-by: Andrew Mason --- doc/viper/viper.md | 35 +++++++++++++++++++++++------------ go/viperutil/config.go | 3 ++- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index a4a5d962c71..ee01cfaabe1 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -206,30 +206,23 @@ func init() { } ``` -## Auto-Documentation - - +## Config Files -## Config File(s) +`viperutil` provides a few flags that allow binaries to read values from config files in addition to defaults, environment variables and flags. +They are: -All vitess components will support taking a single, static (i.e. not "watched" via `WatchConfig`) configuration file. - -The file will be loaded via `ReadInConfig`. -We will log a warning if no file was found, but will not abort (see [docs][viper_read_in_config_docs]). - -Flags for all binaries: - `--config-path` - Default: `$(pwd)` - EnvVar: `VT_CONFIG_PATH` (parsed exactly like a `$PATH` style shell variable). - FlagType: `StringSlice` - Behavior: Paths for `ReadInConfig` to search. -- `--config-type` (default: "") +- `--config-type` - Default: `""` - EnvVar: `VT_CONFIG_TYPE` - FlagType: `flagutil.StringEnum` - Values: everything contained in `viper.SupportedExts`, case-insensitive. - Behavior: Force viper to use a particular unmarshalling strategy; required if the config file does not have an extension (by default, viper infers the config type from the file extension). -- `--config-name` (default: "vtconfig") +- `--config-name` - Default: `"vtconfig"` - EnvVar: `VT_CONFIG_NAME` - FlagType: `string` @@ -239,6 +232,24 @@ Flags for all binaries: - EnvVar: `VT_CONFIG_FILE` - FlagType: `string` - Behavior: Instructs `ReadInConfig` to search in `ConfigPaths` for explicitly a file with this name. Takes precedence over `ConfigName`. +- `--config-file-not-found-handling` + - Default: `WarnOnConfigFileNotFound` + - EnvVar: (none) + - FlagType: `string` (options: `IgnoreConfigFileNotFound`, `WarnOnConfigFileNotFound`, `ErrorOnConfigFileNotFound`, `ExitOnConfigFileNotFound`) + - Behavior: If viper is unable to locate a config file (based on the other flags here), then `LoadConfig` will: + - `Ignore` => do nothing, return no error. Program values will come entirely from defaults, environment variables and flags. + - `Warn` => log at the WARNING level, but return no error. + - `Error` => log at the ERROR level and return the error back to the caller (usually `servenv`.) + - `Exit` => log at the FATAL level, exiting immediately. + +For more information on how viper searches for config files, see the [documentation][viper_read_in_config_docs]. + +If viper was able to locate and load a config file, `LoadConfig` will then configure the dynamic registry to set up a watch on that file, enabling all dynamic values to pick up changes to that file for the remainder of the program's execution. +If no config file was used, then dynamic values behave exactly like static values (i.e. the dynamic registry copies in the settings loaded into the static registry, but does not set up a file watch). + +## Auto-Documentation + + ## Caveats and Gotchas diff --git a/go/viperutil/config.go b/go/viperutil/config.go index 69482eb569c..81940750ce3 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -67,6 +67,7 @@ var ( configFileNotFoundHandling = Configure( "config.notfound.handling", Options[ConfigFileNotFoundHandling]{ + Default: WarnOnConfigFileNotFound, GetFunc: getHandlingValue, }, ) @@ -95,7 +96,7 @@ func RegisterFlags(fs *pflag.FlagSet) { fs.String("config-name", configName.Default(), "Name of the config file (without extension) to search for.") fs.String("config-file", configFile.Default(), "Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored.") - var h = WarnOnConfigFileNotFound + var h = configFileNotFoundHandling.Default() fs.Var(&h, "config-file-not-found-handling", fmt.Sprintf("Behavior when a config file is not found. (Options: %s)", strings.Join(handlingNames, ", "))) BindFlags(fs, configPaths, configType, configName, configFile) From 4b560ffb97e69d3f9673af429f301d62411c42fa Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 21 Nov 2022 14:41:42 -0500 Subject: [PATCH 057/100] docs are done Signed-off-by: Andrew Mason --- doc/viper/viper.md | 45 ++++++++++++++++++++++++++++++++++++------- go/viperutil/viper.go | 2 -- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index ee01cfaabe1..1ba7f7b0992 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -1,7 +1,5 @@ # Vitess Viper Guidelines -### Preface - ## What is Viper? [`viper`][viper] is a configuration-management library for Go programs. @@ -249,14 +247,47 @@ If no config file was used, then dynamic values behave exactly like static value ## Auto-Documentation - +One of the benefits of all values being created through a single function is that we can pretty easily build tooling to generate documentation for the config values available to a given binary. +The exact formatting can be tweaked, obviously, but as an example, something like: + +``` +{{ .BinaryName }} + +{{ range .Values }} +{{ .Key }}: + - Aliases: {{ join .Aliases ", " }} + - Environment Variables: {{ join .EnvVars ", " }} + {{- if hasFlag $.BinaryName .FlagName }} + - Flag: {{ .FlagName }} + {{ end -}} + {{- if hasDefault . }} + - Default: {{ .Default }} + {{ end -}} +{{ end }} +``` + +If/when we migrate other binaries to cobra, we can figure out how to combine this documntation with cobra's doc-generation tooling (which we use for `vtctldclient` and `vtadmin`). ## Caveats and Gotchas -- [ ] case-(in)sensitivity. -- [ ] `Sub` is split-brain -- [ ] `Unmarshal*` functions rely on `mapstructure` tags, not `json|yaml|...` tags. -- [ ] Any config files/paths added _after_ calling `WatchConfig` will not get picked up. +- Config keys are case-insensitive. +`Foo`, `foo`, `fOo`, and `FOO` will all have the same value. + - **Except** for environment variables, which, when read, are case-sensitive (but the config key they are _bound to_ remains case-insensitive). + For example, if you have `viper.BindEnv("foo", "VT_FOO")`, then `VT_FOO=1 ./myprogram` will set the value to `1`, but `Vt_FoO=1 ./myprogram will not`. + The value, though, can still be read _from_ viper as `Foo`, `foo`, `FOO`, and so on. + +- `Sub` is a split-brain. + The viper docs discuss using the `Sub` method on a viper to extract a subtree of a config to pass to a submodule. + This seems like a good idea, but has some fun surprises. + Each viper maintains its own settings map, and extracting a sub-tree creates a second settings map that is now completely divorced from the parent. + If you were to `parent.Set(key, value)`, the sub-viper will still have the old value. + Furthermore, if the parent was watching a config file for changes, the sub-viper is _not_ watching that file. + + For these reasons, we **strongly** discourage use of `v.Sub`. + +- The `Unmarshal*` functions rely on `mapstructure` tags, not `json|yaml|...` tags. + +- Any config files/paths added _after_ calling `WatchConfig` will not get picked up by that viper, and a viper can only watch a single config file. [viper]: https://github.com/spf13/viper [viper_read_in_config_docs]: https://github.com/spf13/viper#reading-config-files diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index 0106b8e95e8..fa4b8b5f7a4 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -69,8 +69,6 @@ Finally, after a call to `viperutil.LoadConfig` (which is done as a part of For more details, refer to the package documentation, as well as the documents in doc/viper/. - -- TODO: markdown doc under doc/ tree giving an overview. */ package viperutil From 9ad7be5085929032f8ee4aac4c7d8871ef827834 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 21 Nov 2022 14:48:44 -0500 Subject: [PATCH 058/100] go mod tidy Signed-off-by: Andrew Mason --- go.mod | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 140860a499e..dcfe435430f 100644 --- a/go.mod +++ b/go.mod @@ -107,7 +107,9 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 + github.com/mitchellh/mapstructure v1.5.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 + github.com/spf13/jwalterweatherman v1.1.0 github.com/xlab/treeprint v1.2.0 golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 golang.org/x/sync v0.1.0 @@ -164,7 +166,6 @@ require ( github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -179,7 +180,6 @@ require ( github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect From 7e812e0e6c442e62fadee53eb752f9126fae99d2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 21 Nov 2022 14:54:45 -0500 Subject: [PATCH 059/100] =?UTF-8?q?gofmt=20=F0=9F=99=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Mason --- go/viperutil/config.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index 81940750ce3..07ae0390be7 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -106,10 +106,10 @@ func RegisterFlags(fs *pflag.FlagSet) { // config values to use. // // Config searching follows the behavior used by viper [1], namely: -// * --config-file (full path, including extension) if set will be used to the -// exclusion of all other flags. -// * --config-type is required if the config file does not have one of viper's -// supported extensions (.yaml, .yml, .json, and so on) +// - --config-file (full path, including extension) if set will be used to the +// exclusion of all other flags. +// - --config-type is required if the config file does not have one of viper's +// supported extensions (.yaml, .yml, .json, and so on) // // An additional --config-file-not-found-handling flag controls how to treat the // situation where viper cannot find any config files in any of the provided From 5d4b8e06e0ab7307633c0849edfb0b6962d4bfa2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 6 Dec 2022 18:28:23 -0500 Subject: [PATCH 060/100] document Debug func Signed-off-by: Andrew Mason --- go/viperutil/debug/debug.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go/viperutil/debug/debug.go b/go/viperutil/debug/debug.go index ad045a8fa2f..e706511ffcf 100644 --- a/go/viperutil/debug/debug.go +++ b/go/viperutil/debug/debug.go @@ -22,6 +22,8 @@ import ( "vitess.io/vitess/go/viperutil/internal/registry" ) +// Debug provides the Debug functionality normally accessible to a given viper +// instance, but for a combination of the private static and dynamic registries. func Debug() { v := viper.New() _ = v.MergeConfigMap(registry.Static.AllSettings()) From 082d9a9408fdce2a84ad63e53296e191d0c83149 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 6 Dec 2022 18:36:57 -0500 Subject: [PATCH 061/100] document StringEnum types Signed-off-by: Andrew Mason --- go/flagutil/enum.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/go/flagutil/enum.go b/go/flagutil/enum.go index d4b0e9f1f38..42f3b6d4f5a 100644 --- a/go/flagutil/enum.go +++ b/go/flagutil/enum.go @@ -23,7 +23,11 @@ import ( "strings" ) -// TODO: docs for all these. +// StringEnum provides a string-like flag value that raises an error if given a +// value not in the set of allowed choices. +// +// This parse-time validation can be case-sensitive or not, depending on which +// constructor (NewStringEnum vs NewCaseInsensitiveStringEnum) was used. type StringEnum struct { name string val string @@ -34,12 +38,22 @@ type StringEnum struct { choiceMapper func(string) string } +// ErrInvalidChoice is returned when parsing a value that is not a valid choice +// for the StringEnum flag. var ErrInvalidChoice = errors.New("invalid choice for enum") +// NewStringEnum returns a new string enum flag with the given name, default, +// and choices. +// +// Parse-time validation is case-sensitive. func NewStringEnum(name string, initialValue string, choices []string) *StringEnum { return newStringEnum(name, initialValue, choices, false) } +// NewCaseInsensitiveStringEnum returns a new string enum flag with the given +// name, default, and choices. +// +// Parse-time validation is case-insensitive. func NewCaseInsensitiveStringEnum(name string, initialValue string, choices []string) *StringEnum { return newStringEnum(name, initialValue, choices, true) } @@ -80,6 +94,7 @@ func newStringEnum(name string, initialValue string, choices []string, caseInsen } } +// Set is part of the pflag.Value interface. func (s *StringEnum) Set(arg string) error { if _, ok := s.choices[s.choiceMapper(arg)]; !ok { msg := "%w (valid choices: %v" @@ -95,5 +110,8 @@ func (s *StringEnum) Set(arg string) error { return nil } +// String is part of the pflag.Value interface. func (s *StringEnum) String() string { return s.val } -func (s *StringEnum) Type() string { return "string" } + +// Type is part of the pflag.Value interface. +func (s *StringEnum) Type() string { return "string" } From 8349ea06f48645e2a3cea0fcb7d949e39c489c4d Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 28 Mar 2023 13:37:00 -0400 Subject: [PATCH 062/100] add debug endpoint to see the full config Signed-off-by: Andrew Mason --- go/viperutil/debug/debug.go | 7 ++- go/viperutil/debug/handler.go | 81 +++++++++++++++++++++++++++++++++++ go/vt/servenv/servenv.go | 3 ++ 3 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 go/viperutil/debug/handler.go diff --git a/go/viperutil/debug/debug.go b/go/viperutil/debug/debug.go index e706511ffcf..eb54bcc33f8 100644 --- a/go/viperutil/debug/debug.go +++ b/go/viperutil/debug/debug.go @@ -25,9 +25,12 @@ import ( // Debug provides the Debug functionality normally accessible to a given viper // instance, but for a combination of the private static and dynamic registries. func Debug() { + combinedRegistry().Debug() +} + +func combinedRegistry() *viper.Viper { v := viper.New() _ = v.MergeConfigMap(registry.Static.AllSettings()) _ = v.MergeConfigMap(registry.Dynamic.AllSettings()) - - v.Debug() + return v } diff --git a/go/viperutil/debug/handler.go b/go/viperutil/debug/handler.go new file mode 100644 index 00000000000..290755f1328 --- /dev/null +++ b/go/viperutil/debug/handler.go @@ -0,0 +1,81 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/spf13/viper" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/slices2" +) + +// HandlerFunc provides an http.HandlerFunc that renders the combined config +// registry (both static and dynamic) for debugging purposes. +// +// By default, this writes the config in viper's "debug" format (what you get +// if you call viper.Debug()). If the query parameter "format" is present, and +// matches one of viper's supported config extensions (case-insensitively), the +// combined config will be written to the response in that format. +// +// Example requests: +// - GET /debug/config +// - GET /debug/config?format=json +// - GET /debug/config?format=yaml +func HandlerFunc(w http.ResponseWriter, r *http.Request) { + if err := acl.CheckAccessHTTP(r, acl.DEBUGGING); err != nil { + acl.SendError(w, err) + return + } + + v := combinedRegistry() + format := strings.ToLower(r.URL.Query().Get("format")) + switch { + case format == "": + v.DebugTo(w) + case slices2.Any(viper.SupportedExts, func(ext string) bool { return ext == format }): + // Got a supported format; write the config to a tempfile in that format, + // then copy it to the response. + // + // (Sadly, viper does not yet have a WriteConfigTo(w io.Writer), so we have + // to do this little hacky workaround). + v.SetConfigType(format) + tmp, err := os.CreateTemp("", "viper_debug") + if err != nil { + http.Error(w, fmt.Sprintf("failed to render config to tempfile: %s", err), http.StatusInternalServerError) + return + } + defer os.Remove(tmp.Name()) + + if err := v.WriteConfigAs(tmp.Name()); err != nil { + http.Error(w, fmt.Sprintf("failed to render config to tempfile: %s", err), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, tmp); err != nil { + http.Error(w, fmt.Sprintf("failed to write rendered config: %s", err), http.StatusInternalServerError) + return + } + default: + http.Error(w, fmt.Sprintf("unsupported config format %s", format), http.StatusBadRequest) + } +} diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index c073c199a97..7525031eac3 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -45,6 +45,7 @@ import ( "vitess.io/vitess/go/stats" "vitess.io/vitess/go/trace" "vitess.io/vitess/go/viperutil" + viperdebug "vitess.io/vitess/go/viperutil/debug" "vitess.io/vitess/go/vt/grpccommon" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" @@ -344,6 +345,7 @@ func ParseFlags(cmd string) { if err := viperutil.LoadConfig(); err != nil { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) } + HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) logutil.PurgeLogs() } @@ -380,6 +382,7 @@ func ParseFlagsWithArgs(cmd string) []string { if err := viperutil.LoadConfig(); err != nil { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) } + HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) logutil.PurgeLogs() From 2246d6ab969e97ee44eed8ad9fef129647dceb1a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 10 May 2023 13:44:58 -0400 Subject: [PATCH 063/100] add registry.Combined Signed-off-by: Andrew Mason --- go/viperutil/debug/debug.go | 11 +---------- go/viperutil/debug/handler.go | 3 ++- go/viperutil/internal/registry/registry.go | 10 ++++++++++ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/go/viperutil/debug/debug.go b/go/viperutil/debug/debug.go index eb54bcc33f8..aa6acbd46ba 100644 --- a/go/viperutil/debug/debug.go +++ b/go/viperutil/debug/debug.go @@ -17,20 +17,11 @@ limitations under the License. package debug import ( - "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/internal/registry" ) // Debug provides the Debug functionality normally accessible to a given viper // instance, but for a combination of the private static and dynamic registries. func Debug() { - combinedRegistry().Debug() -} - -func combinedRegistry() *viper.Viper { - v := viper.New() - _ = v.MergeConfigMap(registry.Static.AllSettings()) - _ = v.MergeConfigMap(registry.Dynamic.AllSettings()) - return v + registry.Combined().Debug() } diff --git a/go/viperutil/debug/handler.go b/go/viperutil/debug/handler.go index 290755f1328..f8d1c851731 100644 --- a/go/viperutil/debug/handler.go +++ b/go/viperutil/debug/handler.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/acl" "vitess.io/vitess/go/slices2" + "vitess.io/vitess/go/viperutil/internal/registry" ) // HandlerFunc provides an http.HandlerFunc that renders the combined config @@ -47,7 +48,7 @@ func HandlerFunc(w http.ResponseWriter, r *http.Request) { return } - v := combinedRegistry() + v := registry.Combined() format := strings.ToLower(r.URL.Query().Get("format")) switch { case format == "": diff --git a/go/viperutil/internal/registry/registry.go b/go/viperutil/internal/registry/registry.go index c8b03ad4e54..1250497e76c 100644 --- a/go/viperutil/internal/registry/registry.go +++ b/go/viperutil/internal/registry/registry.go @@ -47,3 +47,13 @@ type Bindable interface { RegisterAlias(alias string, key string) SetDefault(key string, value any) } + +// Combined returns a viper combining the Static and Dynamic registries. +func Combined() *viper.Viper { + v := viper.New() + _ = v.MergeConfigMap(Static.AllSettings()) + _ = v.MergeConfigMap(Dynamic.AllSettings()) + + v.SetConfigFile(Static.ConfigFileUsed()) + return v +} From 60d0ea1029f922397e29d7a81f24c78ce1c094a8 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 10 May 2023 13:45:57 -0400 Subject: [PATCH 064/100] add Set to Value interface and implementations Signed-off-by: Andrew Mason --- go/viperutil/internal/sync/sync.go | 9 +++++++++ go/viperutil/internal/value/value.go | 8 ++++++++ go/viperutil/value.go | 7 +++++++ 3 files changed, 24 insertions(+) diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index 22d51c4afde..e4e07c4e389 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -53,6 +53,15 @@ func New() *Viper { } } +// Set sets the given key to the given value, in both the disk and live vipers. +func (v *Viper) Set(key string, value any) { + v.m.Lock() + defer v.m.Unlock() + + v.disk.Set(key, value) + v.live.Set(key, value) +} + // ErrDuplicateWatch is returned when Watch is called on a synced Viper which // has already started a watch. var ErrDuplicateWatch = errors.New("duplicate watch") diff --git a/go/viperutil/internal/value/value.go b/go/viperutil/internal/value/value.go index 344f88ed967..fbc754cb993 100644 --- a/go/viperutil/internal/value/value.go +++ b/go/viperutil/internal/value/value.go @@ -136,6 +136,10 @@ func (val *Static[T]) Registry() registry.Bindable { return registry.Static } +func (val *Static[T]) Set(v T) { + registry.Static.Set(val.KeyName, v) +} + // Dynamic is a dynamic value. Dynamic values register to the Dynamic registry, // and respond to changes to watched config files. Their Get() methods will // return whatever value is currently live in the config, in a threadsafe @@ -159,3 +163,7 @@ func NewDynamic[T any](base *Base[T]) *Dynamic[T] { func (val *Dynamic[T]) Registry() registry.Bindable { return registry.Dynamic } + +func (val *Dynamic[T]) Set(v T) { + registry.Dynamic.Set(val.KeyName, v) +} diff --git a/go/viperutil/value.go b/go/viperutil/value.go index 3fc8b6b6a7d..9d5f57938d1 100644 --- a/go/viperutil/value.go +++ b/go/viperutil/value.go @@ -39,6 +39,13 @@ type Value[T any] interface { // never change after the initial config load. For dynamic implementations, // this may change throughout the lifetime of the vitess process. Get() T + // Set sets the underlying value. For both static and dynamic + // implementations, this is reflected in subsequent calls to Get. + // + // If a config file was loaded, changes will be persisted back to the + // config file in the background, governed by the behavior of the + // --config-persistence-min-interval flag. + Set(v T) // Default returns the default value configured for this Value. For both // static and dynamic implementations, it should never change. Default() T From 79d06aebc0e6c356fcc88ea1378157c2632515c2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 10 May 2023 13:57:57 -0400 Subject: [PATCH 065/100] support re-persistence of in-memory changes for existing debugenv APIs Signed-off-by: Andrew Mason --- go/viperutil/config.go | 112 +++++++++++++++++++++++++++++++++++++-- go/vt/servenv/servenv.go | 8 ++- 2 files changed, 114 insertions(+), 6 deletions(-) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index 07ae0390be7..ad77c4f6db4 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -17,11 +17,13 @@ limitations under the License. package viperutil import ( + "context" "fmt" "os" "reflect" "sort" "strings" + "time" "github.com/mitchellh/mapstructure" "github.com/spf13/pflag" @@ -71,6 +73,16 @@ var ( GetFunc: getHandlingValue, }, ) + configPersistenceMinInterval = Configure( + "config.persistence.min_interval", + Options[time.Duration]{ + Default: time.Second, + EnvVars: []string{"VT_CONFIG_PERSISTENCE_MIN_INTERVAL"}, + FlagName: "config-persistence-min-interval", + }, + ) + + persistCh chan struct{} ) func init() { @@ -95,6 +107,7 @@ func RegisterFlags(fs *pflag.FlagSet) { fs.String("config-type", configType.Default(), "Config file type (omit to infer config type from file extension).") fs.String("config-name", configName.Default(), "Name of the config file (without extension) to search for.") fs.String("config-file", configFile.Default(), "Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored.") + fs.Duration("config-persistence-min-interval", configPersistenceMinInterval.Default(), "minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done).") var h = configFileNotFoundHandling.Default() fs.Var(&h, "config-file-not-found-handling", fmt.Sprintf("Behavior when a config file is not found. (Options: %s)", strings.Join(handlingNames, ", "))) @@ -118,10 +131,16 @@ func RegisterFlags(fs *pflag.FlagSet) { // environment variables alone, and not use config files at all). // // If a config file is successfully loaded, then the dynamic registry will also -// start watching that file for changes. +// start watching that file for changes. In addition, in-memory changes to the +// config (for example, from a vtgate or vttablet's debugenv) will be persisted +// back to disk, with writes occuring no more frequently than the +// --config-persistence-min-interval flag. +// +// A cancel function is returned to stop the re-persistence background thread, +// if one was started. // // [1]: https://github.com/spf13/viper#reading-config-files. -func LoadConfig() error { +func LoadConfig() (context.CancelFunc, error) { var err error switch file := configFile.Get(); file { case "": @@ -143,6 +162,7 @@ func LoadConfig() error { err = registry.Static.ReadInConfig() } + usingConfigFile := true if err != nil { if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { msg := "Failed to read in config %s: %s" @@ -152,6 +172,7 @@ func LoadConfig() error { fallthrough // after warning, ignore the error case IgnoreConfigFileNotFound: err = nil + usingConfigFile = false case ErrorOnConfigFileNotFound: log.ERROR(msg, registry.Static.ConfigFileUsed(), nferr.Error()) case ExitOnConfigFileNotFound: @@ -161,10 +182,93 @@ func LoadConfig() error { } if err != nil { - return err + return nil, err + } + + if err := registry.Dynamic.Watch(registry.Static); err != nil { + return nil, err + } + + persistCtx, persistCancel := context.WithCancel(context.Background()) + if usingConfigFile { + persistCh = make(chan struct{}, 1) + go persistChanges(persistCtx, persistCh) + } + + return persistCancel, nil +} + +func persistChanges(ctx context.Context, ch <-chan struct{}) { + // We want the timer to be nil for the first persistence, and only after + // start staggerring future writes by the interval. + var timer *time.Timer + persistConfig := func() { + if err := registry.Combined().WriteConfig(); err != nil { + // If we failed to persist, don't wait the entire interval before + // writing again, instead writing immediately on the next request. + if timer != nil { + if !timer.Stop() { + <-timer.C + } + + timer = nil + } + } + + interval := configPersistenceMinInterval.Get() + if interval == 0 { + return + } + + switch timer { + case nil: + timer = time.NewTimer(configPersistenceMinInterval.Get()) + default: + timer.Reset(configPersistenceMinInterval.Get()) + } + } + + for { + select { + case <-ctx.Done(): + return + case <-ch: + if timer == nil { + persistConfig() + continue + } + select { + case <-ctx.Done(): + return + case <-timer.C: + persistConfig() + } + } + } +} + +// Set sets the value in-memory and notifies the persistence goroutine that it +// needs to write to disk. +func Set[T any](v Value[T], val T) { + v.Set(val) + NotifyConfigChange() +} + +// NotifyConfigChange sends a notification that something in the global config +// was changed at runtime (i.e. **not** from a watched config change), and needs +// to be persisted back to disk. +// +// If no config file was loaded initially, this is a no-op, and no persistence +// is done. +func NotifyConfigChange() { + if persistCh == nil { + return } - return registry.Dynamic.Watch(registry.Static) + select { + case persistCh <- struct{}{}: + default: + } } // NotifyConfigReload adds a subscription that the dynamic registry will attempt diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index 7525031eac3..0d58026aaa7 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -342,9 +342,11 @@ func ParseFlags(cmd string) { log.Exitf("%s doesn't take any positional arguments, got '%s'", cmd, strings.Join(args, " ")) } - if err := viperutil.LoadConfig(); err != nil { + watchCancel, err := viperutil.LoadConfig() + if err != nil { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) } + OnTerm(watchCancel) HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) logutil.PurgeLogs() @@ -379,9 +381,11 @@ func ParseFlagsWithArgs(cmd string) []string { log.Exitf("%s expected at least one positional argument", cmd) } - if err := viperutil.LoadConfig(); err != nil { + watchCancel, err := viperutil.LoadConfig() + if err != nil { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) } + OnTerm(watchCancel) HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) logutil.PurgeLogs() From 124a74515f9212ed8b6abd6d6618d5128da40f21 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 14 May 2023 14:12:39 -0400 Subject: [PATCH 066/100] bugfixes to config persistence Signed-off-by: Andrew Mason --- go/viperutil/config.go | 82 +------------------- go/viperutil/internal/config/config.go | 103 +++++++++++++++++++++++++ go/viperutil/internal/sync/sync.go | 27 ++++++- go/viperutil/internal/value/value.go | 2 + go/viperutil/value.go | 6 +- 5 files changed, 137 insertions(+), 83 deletions(-) create mode 100644 go/viperutil/internal/config/config.go diff --git a/go/viperutil/config.go b/go/viperutil/config.go index ad77c4f6db4..dc5c02fc77e 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -30,6 +30,7 @@ import ( "github.com/spf13/viper" "vitess.io/vitess/go/viperutil/funcs" + "vitess.io/vitess/go/viperutil/internal/config" "vitess.io/vitess/go/viperutil/internal/log" "vitess.io/vitess/go/viperutil/internal/registry" "vitess.io/vitess/go/viperutil/internal/value" @@ -81,8 +82,6 @@ var ( FlagName: "config-persistence-min-interval", }, ) - - persistCh chan struct{} ) func init() { @@ -189,86 +188,11 @@ func LoadConfig() (context.CancelFunc, error) { return nil, err } - persistCtx, persistCancel := context.WithCancel(context.Background()) if usingConfigFile { - persistCh = make(chan struct{}, 1) - go persistChanges(persistCtx, persistCh) + return config.PersistChanges(context.Background(), configPersistenceMinInterval.Get()), nil } - return persistCancel, nil -} - -func persistChanges(ctx context.Context, ch <-chan struct{}) { - // We want the timer to be nil for the first persistence, and only after - // start staggerring future writes by the interval. - var timer *time.Timer - persistConfig := func() { - if err := registry.Combined().WriteConfig(); err != nil { - // If we failed to persist, don't wait the entire interval before - // writing again, instead writing immediately on the next request. - if timer != nil { - if !timer.Stop() { - <-timer.C - } - - timer = nil - } - } - - interval := configPersistenceMinInterval.Get() - if interval == 0 { - return - } - - switch timer { - case nil: - timer = time.NewTimer(configPersistenceMinInterval.Get()) - default: - timer.Reset(configPersistenceMinInterval.Get()) - } - } - - for { - select { - case <-ctx.Done(): - return - case <-ch: - if timer == nil { - persistConfig() - continue - } - select { - case <-ctx.Done(): - return - case <-timer.C: - persistConfig() - } - } - } -} - -// Set sets the value in-memory and notifies the persistence goroutine that it -// needs to write to disk. -func Set[T any](v Value[T], val T) { - v.Set(val) - NotifyConfigChange() -} - -// NotifyConfigChange sends a notification that something in the global config -// was changed at runtime (i.e. **not** from a watched config change), and needs -// to be persisted back to disk. -// -// If no config file was loaded initially, this is a no-op, and no persistence -// is done. -func NotifyConfigChange() { - if persistCh == nil { - return - } - - select { - case persistCh <- struct{}{}: - default: - } + return nil, nil } // NotifyConfigReload adds a subscription that the dynamic registry will attempt diff --git a/go/viperutil/internal/config/config.go b/go/viperutil/internal/config/config.go new file mode 100644 index 00000000000..93ab0fe0e98 --- /dev/null +++ b/go/viperutil/internal/config/config.go @@ -0,0 +1,103 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "context" + "time" + + "vitess.io/vitess/go/viperutil/internal/log" + "vitess.io/vitess/go/viperutil/internal/registry" +) + +var ( + ch chan struct{} +) + +func PersistChanges(ctx context.Context, minWaitInterval time.Duration) context.CancelFunc { + if ch != nil { + panic("PersistChanges already called") + } + + ch = make(chan struct{}, 1) + + var timer *time.Timer + if minWaitInterval > 0 { + timer = time.NewTimer(minWaitInterval) + } + + persistOnce := func() { + if err := registry.Dynamic.WriteConfig(); err != nil { + log.ERROR("failed to persist config changes back to disk: %s", err.Error()) + // If we failed to persist, don't wait the entire interval before + // writing again, instead writing immediately on the next request. + if timer != nil { + if !timer.Stop() { + <-timer.C + } + + timer = nil + } + } + + switch { + case minWaitInterval == 0: + return + case timer == nil: + timer = time.NewTimer(minWaitInterval) + default: + timer.Reset(minWaitInterval) + } + } + + ctx, cancel := context.WithCancel(ctx) + go func() { + defer close(ch) + + for { + select { + case <-ctx.Done(): + return + case <-ch: + if timer == nil { + persistOnce() + continue + } + + select { + case <-ctx.Done(): + return + case <-timer.C: + persistOnce() + } + } + } + }() + + return cancel +} + +func NotifyChanged() { + if ch == nil { + return + } + + select { + case ch <- struct{}{}: + default: + } +} diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index e4e07c4e389..7589eca9adf 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -55,10 +55,19 @@ func New() *Viper { // Set sets the given key to the given value, in both the disk and live vipers. func (v *Viper) Set(key string, value any) { + m, ok := v.keys[key] + if !ok { + return + } + + m.Lock() + defer m.Unlock() + v.m.Lock() defer v.m.Unlock() - v.disk.Set(key, value) + // We must not update v.disk here; explicit calls to Set will supercede all + // future config reloads. v.live.Set(key, value) } @@ -118,6 +127,22 @@ func (v *Viper) Watch(static *viper.Viper) error { return nil } +// WriteConfig writes the live viper config back to disk. +func (v *Viper) WriteConfig() error { + for _, m := range v.keys { + m.Lock() + // This won't fire until after the config has been written. + defer m.Unlock() + } + + v.m.Lock() + defer v.m.Unlock() + + v.live.SetConfigFile(v.disk.ConfigFileUsed()) + + return v.live.WriteConfig() +} + // Notify adds a subscription that this synced viper will attempt to notify on // config changes, after the updated config has been copied over from disk to // live. diff --git a/go/viperutil/internal/value/value.go b/go/viperutil/internal/value/value.go index fbc754cb993..efe60881c4e 100644 --- a/go/viperutil/internal/value/value.go +++ b/go/viperutil/internal/value/value.go @@ -23,6 +23,7 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" + "vitess.io/vitess/go/viperutil/internal/config" "vitess.io/vitess/go/viperutil/internal/registry" "vitess.io/vitess/go/viperutil/internal/sync" ) @@ -166,4 +167,5 @@ func (val *Dynamic[T]) Registry() registry.Bindable { func (val *Dynamic[T]) Set(v T) { registry.Dynamic.Set(val.KeyName, v) + config.NotifyChanged() } diff --git a/go/viperutil/value.go b/go/viperutil/value.go index 9d5f57938d1..214b0299a3c 100644 --- a/go/viperutil/value.go +++ b/go/viperutil/value.go @@ -42,9 +42,9 @@ type Value[T any] interface { // Set sets the underlying value. For both static and dynamic // implementations, this is reflected in subsequent calls to Get. // - // If a config file was loaded, changes will be persisted back to the - // config file in the background, governed by the behavior of the - // --config-persistence-min-interval flag. + // If a config file was loaded, changes to dynamic values will be persisted + // back to the config file in the background, governed by the behavior of + // the --config-persistence-min-interval flag. Set(v T) // Default returns the default value configured for this Value. For both // static and dynamic implementations, it should never change. From 625135e3d015bc1ee467113d4eb198111dad615a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 14 May 2023 14:13:37 -0400 Subject: [PATCH 067/100] fix sync loading to work with re-persistence Signed-off-by: Andrew Mason --- go/viperutil/internal/sync/sync.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index 7589eca9adf..ac7cc5815ad 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -172,6 +172,10 @@ func (v *Viper) loadFromDisk() { v.m.Lock() defer v.m.Unlock() + // Reset v.live so explicit Set calls don't win over what's just changed on + // disk. + v.live = viper.New() + // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an // older version of viper it used to actually handle errors, but now it // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. @@ -207,12 +211,10 @@ func AdaptGetter[T any](key string, getter func(v *viper.Viper) func(key string) var m sync.RWMutex v.keys[key] = &m - boundGet := getter(v.live) - return func(key string) T { m.RLock() defer m.RUnlock() - return boundGet(key) + return getter(v.live)(key) } } From cf8b229fbe5562a06a51e63b4e2bd33d3b535108 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 14 May 2023 14:14:11 -0400 Subject: [PATCH 068/100] make discovery.min_number_serving_vttablets a dynamic viper value Signed-off-by: Andrew Mason --- go/vt/discovery/replicationlag.go | 30 ++++++++++++++++---------- go/vt/discovery/replicationlag_test.go | 4 ++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 71ab78fd15b..96585254d8a 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -23,14 +23,22 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/servenv" ) var ( // lowReplicationLag defines the duration that replication lag is low enough that the VTTablet is considered healthy. - lowReplicationLag time.Duration - highReplicationLagMinServing time.Duration - minNumTablets int + lowReplicationLag time.Duration + highReplicationLagMinServing time.Duration + minNumTablets = viperutil.Configure( + "discovery.min_number_serving_vttablets", + viperutil.Options[int]{ + FlagName: "min_number_serving_vttablets", + Default: 2, + Dynamic: true, + }, + ) legacyReplicationLagAlgorithm bool ) @@ -41,7 +49,7 @@ func init() { func registerReplicationFlags(fs *pflag.FlagSet) { fs.DurationVar(&lowReplicationLag, "discovery_low_replication_lag", 30*time.Second, "Threshold below which replication lag is considered low enough to be healthy.") fs.DurationVar(&highReplicationLagMinServing, "discovery_high_replication_lag_minimum_serving", 2*time.Hour, "Threshold above which replication lag is considered too high when applying the min_number_serving_vttablets flag.") - fs.IntVar(&minNumTablets, "min_number_serving_vttablets", 2, "The minimum number of vttablets for each replicating tablet_type (e.g. replica, rdonly) that will be continue to be used even with replication lag above discovery_low_replication_lag, but still below discovery_high_replication_lag_minimum_serving.") + fs.Int("min_number_serving_vttablets", minNumTablets.Default(), "The minimum number of vttablets for each replicating tablet_type (e.g. replica, rdonly) that will be continue to be used even with replication lag above discovery_low_replication_lag, but still below discovery_high_replication_lag_minimum_serving.") fs.BoolVar(&legacyReplicationLagAlgorithm, "legacy_replication_lag_algorithm", true, "Use the legacy algorithm when selecting vttablets for serving.") } @@ -67,12 +75,12 @@ func SetHighReplicationLagMinServing(lag time.Duration) { // GetMinNumTablets getter for use by debugenv func GetMinNumTablets() int { - return minNumTablets + return minNumTablets.Get() } // SetMinNumTablets setter for use by debugenv func SetMinNumTablets(numTablets int) { - minNumTablets = numTablets + minNumTablets.Set(numTablets) } // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high @@ -119,7 +127,7 @@ func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHeal res := filterStatsByLagWithLegacyAlgorithm(tabletHealthList) // run the filter again if exactly one tablet is removed, // and we have spare tablets. - if len(res) > minNumTablets && len(res) == len(tabletHealthList)-1 { + if len(res) > minNumTablets.Get() && len(res) == len(tabletHealthList)-1 { res = filterStatsByLagWithLegacyAlgorithm(res) } return res @@ -145,7 +153,7 @@ func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { // Pick those with low replication lag, but at least minNumTablets tablets regardless. res := make([]*TabletHealth, 0, len(list)) for i := 0; i < len(list); i++ { - if !IsReplicationLagHigh(list[i].ts) || i < minNumTablets { + if !IsReplicationLagHigh(list[i].ts) || i < minNumTablets.Get() { res = append(res, list[i].ts) } } @@ -186,7 +194,7 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*Ta res = append(res, ts) } } - if len(res) >= minNumTablets { + if len(res) >= minNumTablets.Get() { return res } // return at least minNumTablets tablets to avoid over loading, @@ -219,8 +227,8 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*Ta sort.Sort(byReplag(snapshots)) // Pick the first minNumTablets tablets. - res = make([]*TabletHealth, 0, minNumTablets) - for i := 0; i < min(minNumTablets, len(snapshots)); i++ { + res = make([]*TabletHealth, 0, minNumTablets.Get()) + for i := 0; i < min(minNumTablets.Get(), len(snapshots)); i++ { res = append(res, snapshots[i].ts) } return res diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 80c3921d052..ad87f95cbde 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -30,7 +30,7 @@ import ( func init() { lowReplicationLag = 30 * time.Second highReplicationLagMinServing = 2 * time.Hour - minNumTablets = 2 + minNumTablets.Set(2) legacyReplicationLagAlgorithm = true } @@ -41,7 +41,7 @@ func testSetLegacyReplicationLagAlgorithm(newLegacy bool) { // testSetMinNumTablets is a test helper function, if this is used by a production code path, something is wrong. func testSetMinNumTablets(newMin int) { - minNumTablets = newMin + minNumTablets.Set(newMin) } func TestFilterByReplicationLagUnhealthy(t *testing.T) { From cbf55eaf7a3076a75b12139f654f2e562801438d Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 15 May 2023 06:39:02 -0400 Subject: [PATCH 069/100] update comments Signed-off-by: Andrew Mason --- go/viperutil/internal/config/config.go | 19 +++++++++++++++++++ go/viperutil/value.go | 3 ++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/go/viperutil/internal/config/config.go b/go/viperutil/internal/config/config.go index 93ab0fe0e98..4cf868c7a71 100644 --- a/go/viperutil/internal/config/config.go +++ b/go/viperutil/internal/config/config.go @@ -28,6 +28,23 @@ var ( ch chan struct{} ) +// PersistChanges starts a background goroutine to persist changes made to the +// dynamic registry in-memory (i.e. the "live" config) back to disk (i.e. the +// "disk" config) are persisted back to disk. It returns a cancel func to stop +// the persist loop, which the caller is responsible for calling during +// shutdown (see package servenv for an example). +// +// This does two things — one which is a nice-to-have, and another which is +// necessary for correctness. +// +// 1. Writing in-memory changes (which usually occur through a request to a +// /debug/env endpoint) ensures they are persisted across process restarts. +// 2. Writing in-memory changes ensures that subsequent modifications to the +// config file do not clobber those changes. Because viper loads the entire +// config on-change, rather than an incremental (diff) load, if a user were to +// edit an unrelated key (keyA) in the file, and we did not persist the +// in-memory change (keyB), then future calls to keyB.Get() would return the +// older value. func PersistChanges(ctx context.Context, minWaitInterval time.Duration) context.CancelFunc { if ch != nil { panic("PersistChanges already called") @@ -91,6 +108,8 @@ func PersistChanges(ctx context.Context, minWaitInterval time.Duration) context. return cancel } +// NotifyChanged signals to the persist loop started by PersistChanges() that +// something in the config has changed, and should be persisted soon. func NotifyChanged() { if ch == nil { return diff --git a/go/viperutil/value.go b/go/viperutil/value.go index 214b0299a3c..e6efdc74888 100644 --- a/go/viperutil/value.go +++ b/go/viperutil/value.go @@ -31,7 +31,8 @@ var ( // // N.B. the embedded value.Registerable interface is necessary only for // BindFlags and other mechanisms of binding Values to the internal registries -// to work. Users of Value objects should only need to call Get() and Default(). +// to work. Users of Value objects should only need to call Get(), Set(v T), and +// Default(). type Value[T any] interface { value.Registerable From 8ff1866b2804758feaac0c31035a71ddbccc1aa9 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 15 May 2023 06:46:48 -0400 Subject: [PATCH 070/100] configure the rest of discovery replicationlag settings Signed-off-by: Andrew Mason --- go/vt/discovery/replicationlag.go | 56 +++++++++++++++++++------- go/vt/discovery/replicationlag_test.go | 8 ++-- go/vt/discovery/tablet_health_check.go | 2 +- 3 files changed, 47 insertions(+), 19 deletions(-) diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 96585254d8a..24d2c0114d6 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -28,10 +28,25 @@ import ( ) var ( + configKey = viperutil.KeyPrefixFunc("discovery") // lowReplicationLag defines the duration that replication lag is low enough that the VTTablet is considered healthy. - lowReplicationLag time.Duration - highReplicationLagMinServing time.Duration - minNumTablets = viperutil.Configure( + lowReplicationLag = viperutil.Configure( + configKey("low_replication_lag"), + viperutil.Options[time.Duration]{ + FlagName: "discovery_low_replication_lag", + Default: 30 * time.Second, + Dynamic: true, + }, + ) + highReplicationLagMinServing = viperutil.Configure( + configKey("high_replication_lag"), + viperutil.Options[time.Duration]{ + FlagName: "discovery_high_replication_lag_minimum_serving", + Default: 2 * time.Hour, + Dynamic: true, + }, + ) + minNumTablets = viperutil.Configure( "discovery.min_number_serving_vttablets", viperutil.Options[int]{ FlagName: "min_number_serving_vttablets", @@ -39,7 +54,13 @@ var ( Dynamic: true, }, ) - legacyReplicationLagAlgorithm bool + legacyReplicationLagAlgorithm = viperutil.Configure( + configKey("legacy_replication_lag_algorithm"), + viperutil.Options[bool]{ + FlagName: "legacy_replication_lag_algorithm", + Default: true, + }, + ) ) func init() { @@ -47,30 +68,37 @@ func init() { } func registerReplicationFlags(fs *pflag.FlagSet) { - fs.DurationVar(&lowReplicationLag, "discovery_low_replication_lag", 30*time.Second, "Threshold below which replication lag is considered low enough to be healthy.") - fs.DurationVar(&highReplicationLagMinServing, "discovery_high_replication_lag_minimum_serving", 2*time.Hour, "Threshold above which replication lag is considered too high when applying the min_number_serving_vttablets flag.") + fs.Duration("discovery_low_replication_lag", lowReplicationLag.Default(), "Threshold below which replication lag is considered low enough to be healthy.") + fs.Duration("discovery_high_replication_lag_minimum_serving", highReplicationLagMinServing.Default(), "Threshold above which replication lag is considered too high when applying the min_number_serving_vttablets flag.") fs.Int("min_number_serving_vttablets", minNumTablets.Default(), "The minimum number of vttablets for each replicating tablet_type (e.g. replica, rdonly) that will be continue to be used even with replication lag above discovery_low_replication_lag, but still below discovery_high_replication_lag_minimum_serving.") - fs.BoolVar(&legacyReplicationLagAlgorithm, "legacy_replication_lag_algorithm", true, "Use the legacy algorithm when selecting vttablets for serving.") + fs.Bool("legacy_replication_lag_algorithm", legacyReplicationLagAlgorithm.Default(), "Use the legacy algorithm when selecting vttablets for serving.") + + viperutil.BindFlags(fs, + lowReplicationLag, + highReplicationLagMinServing, + minNumTablets, + legacyReplicationLagAlgorithm, + ) } // GetLowReplicationLag getter for use by debugenv func GetLowReplicationLag() time.Duration { - return lowReplicationLag + return lowReplicationLag.Get() } // SetLowReplicationLag setter for use by debugenv func SetLowReplicationLag(lag time.Duration) { - lowReplicationLag = lag + lowReplicationLag.Set(lag) } // GetHighReplicationLagMinServing getter for use by debugenv func GetHighReplicationLagMinServing() time.Duration { - return highReplicationLagMinServing + return highReplicationLagMinServing.Get() } // SetHighReplicationLagMinServing setter for use by debugenv func SetHighReplicationLagMinServing(lag time.Duration) { - highReplicationLagMinServing = lag + highReplicationLagMinServing.Set(lag) } // GetMinNumTablets getter for use by debugenv @@ -86,13 +114,13 @@ func SetMinNumTablets(numTablets int) { // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. func IsReplicationLagHigh(tabletHealth *TabletHealth) bool { - return float64(tabletHealth.Stats.ReplicationLagSeconds) > lowReplicationLag.Seconds() + return float64(tabletHealth.Stats.ReplicationLagSeconds) > lowReplicationLag.Get().Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacytabletHealth refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { - return float64(tabletHealth.Stats.ReplicationLagSeconds) > highReplicationLagMinServing.Seconds() + return float64(tabletHealth.Stats.ReplicationLagSeconds) > highReplicationLagMinServing.Get().Seconds() } // FilterStatsByReplicationLag filters the list of TabletHealth by TabletHealth.Stats.ReplicationLagSeconds. @@ -121,7 +149,7 @@ func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { // - degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHealth { - if !legacyReplicationLagAlgorithm { + if !legacyReplicationLagAlgorithm.Get() { return filterStatsByLag(tabletHealthList) } res := filterStatsByLagWithLegacyAlgorithm(tabletHealthList) diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index ad87f95cbde..9c047469fb2 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -28,15 +28,15 @@ import ( ) func init() { - lowReplicationLag = 30 * time.Second - highReplicationLagMinServing = 2 * time.Hour + lowReplicationLag.Set(30 * time.Second) + highReplicationLagMinServing.Set(2 * time.Hour) minNumTablets.Set(2) - legacyReplicationLagAlgorithm = true + legacyReplicationLagAlgorithm.Set(true) } // testSetLegacyReplicationLagAlgorithm is a test helper function, if this is used by a production code path, something is wrong. func testSetLegacyReplicationLagAlgorithm(newLegacy bool) { - legacyReplicationLagAlgorithm = newLegacy + legacyReplicationLagAlgorithm.Set(newLegacy) } // testSetMinNumTablets is a test helper function, if this is used by a production code path, something is wrong. diff --git a/go/vt/discovery/tablet_health_check.go b/go/vt/discovery/tablet_health_check.go index b3cc06e2d10..95821db88a2 100644 --- a/go/vt/discovery/tablet_health_check.go +++ b/go/vt/discovery/tablet_health_check.go @@ -212,7 +212,7 @@ func (thc *tabletHealthCheck) isTrivialReplagChange(newStats *query.RealtimeStat } // Skip replag filter when replag remains in the low rep lag range, // which should be the case majority of the time. - lowRepLag := lowReplicationLag.Seconds() + lowRepLag := lowReplicationLag.Get().Seconds() oldRepLag := float64(thc.Stats.ReplicationLagSeconds) newRepLag := float64(newStats.ReplicationLagSeconds) if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { From fa2d48f6f4650e5a841a13a4ff533f6fcd583bd3 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 15 May 2023 06:48:38 -0400 Subject: [PATCH 071/100] remove example code Signed-off-by: Andrew Mason --- go/vt/topo/locks.go | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/go/vt/topo/locks.go b/go/vt/topo/locks.go index e40792fcb9c..8d30d85e891 100644 --- a/go/vt/topo/locks.go +++ b/go/vt/topo/locks.go @@ -29,7 +29,6 @@ import ( _flag "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/trace" - "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/servenv" @@ -44,15 +43,6 @@ var ( // shard / keyspace lock can be acquired for. LockTimeout = 45 * time.Second - remoteOperationTimeout = viperutil.Configure( - "topo.remote_operation_timeout", - viperutil.Options[time.Duration]{ - Default: 30 * time.Second, - FlagName: "remote_operation_timeout", - Dynamic: true, - }, - ) - // RemoteOperationTimeout is used for operations where we have to // call out to another process. // Used for RPC calls (including topo server calls) @@ -76,30 +66,10 @@ func init() { for _, cmd := range FlagBinaries { servenv.OnParseFor(cmd, registerTopoLockFlags) } - - ch := make(chan struct{}, 1) - go func() { - for range ch { - log.Errorf("topo.RemoteOperationTimeout updated; new value: %v", remoteOperationTimeout.Get()) - } - - log.Errorf("topo settings channel closed") - }() - - servenv.OnTerm(func() { - close(ch) - }) - - viperutil.NotifyConfigReload(ch) - - log.Errorf("initialized config watcher for topo settings") - log.Errorf("initial topo.RemoteOperationTimeout: %v", remoteOperationTimeout.Get()) } func registerTopoLockFlags(fs *pflag.FlagSet) { fs.DurationVar(&RemoteOperationTimeout, "remote_operation_timeout", RemoteOperationTimeout, "time to wait for a remote operation") - viperutil.BindFlags(fs, remoteOperationTimeout) - fs.DurationVar(&LockTimeout, "lock-timeout", LockTimeout, "Maximum time for which a shard/keyspace lock can be acquired for") } From fe2792de02462d1a90c2c23172602d3e49c98db2 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 10:01:15 -0400 Subject: [PATCH 072/100] fix bindings now that live gets reset Signed-off-by: Andrew Mason --- go/viperutil/internal/sync/sync.go | 8 ++++---- go/vt/discovery/replicationlag.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index ac7cc5815ad..1aab57d8005 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -184,10 +184,10 @@ func (v *Viper) loadFromDisk() { // begin implementation of registry.Bindable for sync.Viper -func (v *Viper) BindEnv(vars ...string) error { return v.live.BindEnv(vars...) } -func (v *Viper) BindPFlag(key string, flag *pflag.Flag) error { return v.live.BindPFlag(key, flag) } -func (v *Viper) RegisterAlias(alias string, key string) { v.live.RegisterAlias(alias, key) } -func (v *Viper) SetDefault(key string, value any) { v.live.SetDefault(key, value) } +func (v *Viper) BindEnv(vars ...string) error { return v.disk.BindEnv(vars...) } +func (v *Viper) BindPFlag(key string, flag *pflag.Flag) error { return v.disk.BindPFlag(key, flag) } +func (v *Viper) RegisterAlias(alias string, key string) { v.disk.RegisterAlias(alias, key) } +func (v *Viper) SetDefault(key string, value any) { v.disk.SetDefault(key, value) } // end implementation of registry.Bindable for sync.Viper diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 24d2c0114d6..8ae168ddff9 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -47,7 +47,7 @@ var ( }, ) minNumTablets = viperutil.Configure( - "discovery.min_number_serving_vttablets", + configKey("min_number_serving_vttablets"), viperutil.Options[int]{ FlagName: "min_number_serving_vttablets", Default: 2, From 432fd6f0f04b7bb375b89d9cbc5998ab4ab85034 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 10:12:51 -0400 Subject: [PATCH 073/100] default to ignore Signed-off-by: Andrew Mason --- go/viperutil/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index dc5c02fc77e..7a3c81ce6d8 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -70,7 +70,7 @@ var ( configFileNotFoundHandling = Configure( "config.notfound.handling", Options[ConfigFileNotFoundHandling]{ - Default: WarnOnConfigFileNotFound, + Default: IgnoreConfigFileNotFound, GetFunc: getHandlingValue, }, ) From fdeb7d43605a491c3e766baa29813fc60c547c38 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 12:07:36 -0400 Subject: [PATCH 074/100] return empty cancel func to not panic Signed-off-by: Andrew Mason --- go/viperutil/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index 7a3c81ce6d8..b1a279c14a2 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -192,7 +192,7 @@ func LoadConfig() (context.CancelFunc, error) { return config.PersistChanges(context.Background(), configPersistenceMinInterval.Get()), nil } - return nil, nil + return func() {}, nil } // NotifyConfigReload adds a subscription that the dynamic registry will attempt From f942159b0ace9b0e2bb94e08cd22272a8d0caee4 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 13:36:28 -0400 Subject: [PATCH 075/100] no double registrations Signed-off-by: Andrew Mason --- go/vt/servenv/servenv.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index 0d58026aaa7..491d004e76e 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -322,6 +322,10 @@ func getFlagHooksFor(cmd string) (hooks []func(fs *pflag.FlagSet)) { return hooks } +// Needed because some tests require multiple parse passes, so we guard against +// that here. +var debugConfigRegisterOnce sync.Once + // ParseFlags initializes flags and handles the common case when no positional // arguments are expected. func ParseFlags(cmd string) { @@ -347,7 +351,9 @@ func ParseFlags(cmd string) { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) } OnTerm(watchCancel) - HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) + debugConfigRegisterOnce.Do(func() { + HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) + }) logutil.PurgeLogs() } @@ -386,7 +392,9 @@ func ParseFlagsWithArgs(cmd string) []string { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) } OnTerm(watchCancel) - HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) + debugConfigRegisterOnce.Do(func() { + HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) + }) logutil.PurgeLogs() From 674c2565afa8ada090b23b692a0334b6c7cad3ed Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 13:38:40 -0400 Subject: [PATCH 076/100] update flag testdata Signed-off-by: Andrew Mason --- go/flags/endtoend/mysqlctl.txt | 142 +++++------ go/flags/endtoend/mysqlctld.txt | 6 + go/flags/endtoend/vtaclcheck.txt | 42 ++-- go/flags/endtoend/vtbackup.txt | 380 +++++++++++++++-------------- go/flags/endtoend/vtctlclient.txt | 86 ++++--- go/flags/endtoend/vtctld.txt | 6 + go/flags/endtoend/vtexplain.txt | 78 +++--- go/flags/endtoend/vtgate.txt | 7 + go/flags/endtoend/vtgr.txt | 172 ++++++------- go/flags/endtoend/vtorc.txt | 178 +++++++------- go/flags/endtoend/vttablet.txt | 6 + go/flags/endtoend/vttestserver.txt | 6 + go/flags/endtoend/zkctl.txt | 40 +-- go/flags/endtoend/zkctld.txt | 42 ++-- 14 files changed, 638 insertions(+), 553 deletions(-) diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index cdd64593649..4910fc97e2a 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -11,71 +11,77 @@ The commands are listed below. Use 'mysqlctl -- {-h, --help}' for comm position Global flags: - --alsologtostderr log to standard error as well as files - --app_idle_timeout duration Idle timeout for app connections (default 1m0s) - --app_pool_size int Size of the connection pool for app connections (default 40) - --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified - --db-credentials-file string db credentials file; send SIGHUP to reload this file - --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") - --db-credentials-vault-addr string URL to Vault server - --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds - --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") - --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable - --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable - --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) - --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate - --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable - --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) - --db_charset string Character set used for this tablet. (default "utf8mb4") - --db_conn_query_info enable parsing and processing of QUERY_OK info fields - --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) - --db_dba_password string db dba password - --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) - --db_dba_user string db dba user userKey (default "vt_dba") - --db_flags uint Flag values as defined by MySQL. - --db_flavor string Flavor overrid. Valid value is FilePos. - --db_host string The host name for the tcp connection. - --db_port int tcp port - --db_server_name string server name of the DB we are connecting to. - --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. - --db_ssl_ca string connection ssl ca - --db_ssl_ca_path string connection ssl ca path - --db_ssl_cert string connection ssl certificate - --db_ssl_key string connection ssl key - --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. - --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. - --dba_idle_timeout duration Idle timeout for dba connections (default 1m0s) - --dba_pool_size int Size of the connection pool for dba connections (default 20) - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --max-stack-size int configure the maximum stack size in bytes (default 67108864) - --mysql_port int MySQL port (default 3306) - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_socket string Path to the mysqld socket file - --mysqlctl_client_protocol string the protocol to use to talk to the mysqlctl server (default "grpc") - --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init - --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) - --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) - --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) - --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. - --pool_hostname_resolve_interval duration if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled) - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --replication_connect_retry duration how long to wait in between replica reconnect attempts. Only precise to the second. (default 10s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice - --socket_file string Local unix socket file to listen on - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class - --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. - --tablet_uid uint32 Tablet UID (default 41983) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --alsologtostderr log to standard error as well as files + --app_idle_timeout duration Idle timeout for app connections (default 1m0s) + --app_pool_size int Size of the connection pool for app connections (default 40) + --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --db-credentials-file string db credentials file; send SIGHUP to reload this file + --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") + --db-credentials-vault-addr string URL to Vault server + --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds + --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") + --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable + --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable + --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) + --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate + --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable + --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) + --db_charset string Character set used for this tablet. (default "utf8mb4") + --db_conn_query_info enable parsing and processing of QUERY_OK info fields + --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) + --db_dba_password string db dba password + --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) + --db_dba_user string db dba user userKey (default "vt_dba") + --db_flags uint Flag values as defined by MySQL. + --db_flavor string Flavor overrid. Valid value is FilePos. + --db_host string The host name for the tcp connection. + --db_port int tcp port + --db_server_name string server name of the DB we are connecting to. + --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. + --db_ssl_ca string connection ssl ca + --db_ssl_ca_path string connection ssl ca path + --db_ssl_cert string connection ssl certificate + --db_ssl_key string connection ssl key + --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. + --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. + --dba_idle_timeout duration Idle timeout for dba connections (default 1m0s) + --dba_pool_size int Size of the connection pool for dba connections (default 20) + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --mysql_port int MySQL port (default 3306) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_socket string Path to the mysqld socket file + --mysqlctl_client_protocol string the protocol to use to talk to the mysqlctl server (default "grpc") + --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init + --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) + --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) + --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) + --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. + --pool_hostname_resolve_interval duration if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled) + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --replication_connect_retry duration how long to wait in between replica reconnect attempts. Only precise to the second. (default 10s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice + --socket_file string Local unix socket file to listen on + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class + --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. + --tablet_uid uint32 Tablet UID (default 41983) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index 09e3c144620..250cbf539d4 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -3,6 +3,12 @@ Usage of mysqlctld: --app_idle_timeout duration Idle timeout for app connections (default 1m0s) --app_pool_size int Size of the connection pool for app connections (default 40) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") --db-credentials-vault-addr string URL to Vault server diff --git a/go/flags/endtoend/vtaclcheck.txt b/go/flags/endtoend/vtaclcheck.txt index 6e2c57db109..b25e22af0d9 100644 --- a/go/flags/endtoend/vtaclcheck.txt +++ b/go/flags/endtoend/vtaclcheck.txt @@ -1,19 +1,25 @@ Usage of vtaclcheck: - --acl-file string The path of the JSON ACL file to check - --alsologtostderr log to standard error as well as files - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --static-auth-file string The path of the auth_server_static JSON file to check - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --acl-file string The path of the JSON ACL file to check + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --static-auth-file string The path of the auth_server_static JSON file to check + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index ded70c2550a..4a4b05b251b 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -1,188 +1,194 @@ Usage of vtbackup: - --allow_first_backup Allow this job to take the first backup of an existing shard. - --alsologtostderr log to standard error as well as files - --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). - --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. - --azblob_backup_container_name string Azure Blob Container Name. - --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) - --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). - --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") - --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) - --backup_storage_compress if set, the backup files will be compressed. (default true) - --backup_storage_implementation string Which backup storage implementation to use for creating and restoring backups. - --backup_storage_number_blocks int if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2) - --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. - --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) - --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) - --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) - --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") - --compression-engine-name string compressor engine used for compression. (default "pargzip") - --compression-level int what level to pass to the compressor. (default 1) - --concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) - --consul_auth_static_file string JSON File to read the topos/tokens from. - --db-credentials-file string db credentials file; send SIGHUP to reload this file - --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") - --db-credentials-vault-addr string URL to Vault server - --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds - --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") - --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable - --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable - --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) - --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate - --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable - --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) - --db_allprivs_password string db allprivs password - --db_allprivs_use_ssl Set this flag to false to make the allprivs connection to not use ssl (default true) - --db_allprivs_user string db allprivs user userKey (default "vt_allprivs") - --db_app_password string db app password - --db_app_use_ssl Set this flag to false to make the app connection to not use ssl (default true) - --db_app_user string db app user userKey (default "vt_app") - --db_appdebug_password string db appdebug password - --db_appdebug_use_ssl Set this flag to false to make the appdebug connection to not use ssl (default true) - --db_appdebug_user string db appdebug user userKey (default "vt_appdebug") - --db_charset string Character set used for this tablet. (default "utf8mb4") - --db_conn_query_info enable parsing and processing of QUERY_OK info fields - --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) - --db_dba_password string db dba password - --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) - --db_dba_user string db dba user userKey (default "vt_dba") - --db_erepl_password string db erepl password - --db_erepl_use_ssl Set this flag to false to make the erepl connection to not use ssl (default true) - --db_erepl_user string db erepl user userKey (default "vt_erepl") - --db_filtered_password string db filtered password - --db_filtered_use_ssl Set this flag to false to make the filtered connection to not use ssl (default true) - --db_filtered_user string db filtered user userKey (default "vt_filtered") - --db_flags uint Flag values as defined by MySQL. - --db_flavor string Flavor overrid. Valid value is FilePos. - --db_host string The host name for the tcp connection. - --db_port int tcp port - --db_repl_password string db repl password - --db_repl_use_ssl Set this flag to false to make the repl connection to not use ssl (default true) - --db_repl_user string db repl user userKey (default "vt_repl") - --db_server_name string server name of the DB we are connecting to. - --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. - --db_ssl_ca string connection ssl ca - --db_ssl_ca_path string connection ssl ca path - --db_ssl_cert string connection ssl certificate - --db_ssl_key string connection ssl key - --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. - --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. - --detach detached mode - run backups detached from the terminal - --disable-redo-log Disable InnoDB redo log during replication-from-primary phase of backup. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --external-compressor string command with arguments to use when compressing a backup. - --external-compressor-extension string extension to use when using an external compressor. - --external-decompressor string command with arguments to use when decompressing a backup. - --file_backup_storage_root string Root directory for the file backup storage. - --gcs_backup_storage_bucket string Google Cloud Storage bucket to use for backups. - --gcs_backup_storage_root string Root prefix for all backup-related object names. - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --incremental_from_pos string Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position - --init_db_name_override string (init parameter) override the name of the db used by vttablet - --init_db_sql_file string path to .sql file to run after mysql_install_db - --init_keyspace string (init parameter) keyspace to use for this tablet - --init_shard string (init parameter) shard to use for this tablet - --initial_backup Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed). - --keep-alive-timeout duration Wait until timeout elapses after a successful backup before shutting down. - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --manifest-external-decompressor string command with arguments to store in the backup manifest when compressing a backup with an external compression engine. - --min_backup_interval duration Only take a new backup if it's been at least this long since the most recent backup. - --min_retention_count int Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made (default 1) - --min_retention_time duration Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups. - --mycnf-file string path to my.cnf, if reading all config params from there - --mycnf_bin_log_path string mysql binlog path - --mycnf_data_dir string data directory for mysql - --mycnf_error_log_path string mysql error log path - --mycnf_general_log_path string mysql general log path - --mycnf_innodb_data_home_dir string Innodb data home directory - --mycnf_innodb_log_group_home_dir string Innodb log group home directory - --mycnf_master_info_file string mysql master.info file - --mycnf_mysql_port int port mysql is listening on - --mycnf_pid_file string mysql pid file - --mycnf_relay_log_index_path string mysql relay log index path - --mycnf_relay_log_info_path string mysql relay log info path - --mycnf_relay_log_path string mysql relay log path - --mycnf_secure_file_priv string mysql path for loading secure files - --mycnf_server_id int mysql server id of the server (if specified, mycnf-file will be ignored) - --mycnf_slow_log_path string mysql slow query log path - --mycnf_socket_file string mysql socket file - --mycnf_tmp_dir string mysql tmp directory - --mysql_port int mysql port (default 3306) - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_socket string path to the mysql socket - --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) - --port int port for the server - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --restart_before_backup Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs. - --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). - --s3_backup_aws_region string AWS region to use. (default "us-east-1") - --s3_backup_aws_retries int AWS request retries. (default -1) - --s3_backup_force_path_style force the s3 path style. - --s3_backup_log_level string determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors. (default "LogOff") - --s3_backup_server_side_encryption string server-side encryption algorithm (e.g., AES256, aws:kms, sse_c:/path/to/key/file). - --s3_backup_storage_bucket string S3 bucket to use for backups. - --s3_backup_storage_root string root prefix for all backup-related object names. - --s3_backup_tls_skip_verify_cert skip the 'certificate is valid' check for SSL connections. - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) - --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --xbstream_restore_flags string Flags to pass to xbstream command during restore. These should be space separated and will be added to the end of the command. These need to match the ones used for backup e.g. --compress / --decompress, --encrypt / --decrypt - --xtrabackup_backup_flags string Flags to pass to backup command. These should be space separated and will be added to the end of the command - --xtrabackup_prepare_flags string Flags to pass to prepare command. These should be space separated and will be added to the end of the command - --xtrabackup_root_path string Directory location of the xtrabackup and xbstream executables, e.g., /usr/bin - --xtrabackup_stream_mode string Which mode to use if streaming, valid values are tar and xbstream. Please note that tar is not supported in XtraBackup 8.0 (default "tar") - --xtrabackup_stripe_block_size uint Size in bytes of each block that gets sent to a given stripe before rotating to the next stripe (default 102400) - --xtrabackup_stripes uint If greater than 0, use data striping across this many destination files to parallelize data transfer and decompression - --xtrabackup_user string User that xtrabackup will use to connect to the database server. This user must have all necessary privileges. For details, please refer to xtrabackup documentation. + --allow_first_backup Allow this job to take the first backup of an existing shard. + --alsologtostderr log to standard error as well as files + --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). + --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_container_name string Azure Blob Container Name. + --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) + --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). + --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") + --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) + --backup_storage_compress if set, the backup files will be compressed. (default true) + --backup_storage_implementation string Which backup storage implementation to use for creating and restoring backups. + --backup_storage_number_blocks int if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2) + --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. + --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) + --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) + --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") + --compression-engine-name string compressor engine used for compression. (default "pargzip") + --compression-level int what level to pass to the compressor. (default 1) + --concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --consul_auth_static_file string JSON File to read the topos/tokens from. + --db-credentials-file string db credentials file; send SIGHUP to reload this file + --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") + --db-credentials-vault-addr string URL to Vault server + --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds + --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") + --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable + --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable + --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) + --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate + --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable + --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) + --db_allprivs_password string db allprivs password + --db_allprivs_use_ssl Set this flag to false to make the allprivs connection to not use ssl (default true) + --db_allprivs_user string db allprivs user userKey (default "vt_allprivs") + --db_app_password string db app password + --db_app_use_ssl Set this flag to false to make the app connection to not use ssl (default true) + --db_app_user string db app user userKey (default "vt_app") + --db_appdebug_password string db appdebug password + --db_appdebug_use_ssl Set this flag to false to make the appdebug connection to not use ssl (default true) + --db_appdebug_user string db appdebug user userKey (default "vt_appdebug") + --db_charset string Character set used for this tablet. (default "utf8mb4") + --db_conn_query_info enable parsing and processing of QUERY_OK info fields + --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) + --db_dba_password string db dba password + --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) + --db_dba_user string db dba user userKey (default "vt_dba") + --db_erepl_password string db erepl password + --db_erepl_use_ssl Set this flag to false to make the erepl connection to not use ssl (default true) + --db_erepl_user string db erepl user userKey (default "vt_erepl") + --db_filtered_password string db filtered password + --db_filtered_use_ssl Set this flag to false to make the filtered connection to not use ssl (default true) + --db_filtered_user string db filtered user userKey (default "vt_filtered") + --db_flags uint Flag values as defined by MySQL. + --db_flavor string Flavor overrid. Valid value is FilePos. + --db_host string The host name for the tcp connection. + --db_port int tcp port + --db_repl_password string db repl password + --db_repl_use_ssl Set this flag to false to make the repl connection to not use ssl (default true) + --db_repl_user string db repl user userKey (default "vt_repl") + --db_server_name string server name of the DB we are connecting to. + --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. + --db_ssl_ca string connection ssl ca + --db_ssl_ca_path string connection ssl ca path + --db_ssl_cert string connection ssl certificate + --db_ssl_key string connection ssl key + --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. + --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. + --detach detached mode - run backups detached from the terminal + --disable-redo-log Disable InnoDB redo log during replication-from-primary phase of backup. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --external-compressor string command with arguments to use when compressing a backup. + --external-compressor-extension string extension to use when using an external compressor. + --external-decompressor string command with arguments to use when decompressing a backup. + --file_backup_storage_root string Root directory for the file backup storage. + --gcs_backup_storage_bucket string Google Cloud Storage bucket to use for backups. + --gcs_backup_storage_root string Root prefix for all backup-related object names. + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --incremental_from_pos string Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position + --init_db_name_override string (init parameter) override the name of the db used by vttablet + --init_db_sql_file string path to .sql file to run after mysql_install_db + --init_keyspace string (init parameter) keyspace to use for this tablet + --init_shard string (init parameter) shard to use for this tablet + --initial_backup Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed). + --keep-alive-timeout duration Wait until timeout elapses after a successful backup before shutting down. + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --manifest-external-decompressor string command with arguments to store in the backup manifest when compressing a backup with an external compression engine. + --min_backup_interval duration Only take a new backup if it's been at least this long since the most recent backup. + --min_retention_count int Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made (default 1) + --min_retention_time duration Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups. + --mycnf-file string path to my.cnf, if reading all config params from there + --mycnf_bin_log_path string mysql binlog path + --mycnf_data_dir string data directory for mysql + --mycnf_error_log_path string mysql error log path + --mycnf_general_log_path string mysql general log path + --mycnf_innodb_data_home_dir string Innodb data home directory + --mycnf_innodb_log_group_home_dir string Innodb log group home directory + --mycnf_master_info_file string mysql master.info file + --mycnf_mysql_port int port mysql is listening on + --mycnf_pid_file string mysql pid file + --mycnf_relay_log_index_path string mysql relay log index path + --mycnf_relay_log_info_path string mysql relay log info path + --mycnf_relay_log_path string mysql relay log path + --mycnf_secure_file_priv string mysql path for loading secure files + --mycnf_server_id int mysql server id of the server (if specified, mycnf-file will be ignored) + --mycnf_slow_log_path string mysql slow query log path + --mycnf_socket_file string mysql socket file + --mycnf_tmp_dir string mysql tmp directory + --mysql_port int mysql port (default 3306) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_socket string path to the mysql socket + --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) + --port int port for the server + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --restart_before_backup Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs. + --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). + --s3_backup_aws_region string AWS region to use. (default "us-east-1") + --s3_backup_aws_retries int AWS request retries. (default -1) + --s3_backup_force_path_style force the s3 path style. + --s3_backup_log_level string determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors. (default "LogOff") + --s3_backup_server_side_encryption string server-side encryption algorithm (e.g., AES256, aws:kms, sse_c:/path/to/key/file). + --s3_backup_storage_bucket string S3 bucket to use for backups. + --s3_backup_storage_root string root prefix for all backup-related object names. + --s3_backup_tls_skip_verify_cert skip the 'certificate is valid' check for SSL connections. + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) + --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --xbstream_restore_flags string Flags to pass to xbstream command during restore. These should be space separated and will be added to the end of the command. These need to match the ones used for backup e.g. --compress / --decompress, --encrypt / --decrypt + --xtrabackup_backup_flags string Flags to pass to backup command. These should be space separated and will be added to the end of the command + --xtrabackup_prepare_flags string Flags to pass to prepare command. These should be space separated and will be added to the end of the command + --xtrabackup_root_path string Directory location of the xtrabackup and xbstream executables, e.g., /usr/bin + --xtrabackup_stream_mode string Which mode to use if streaming, valid values are tar and xbstream. Please note that tar is not supported in XtraBackup 8.0 (default "tar") + --xtrabackup_stripe_block_size uint Size in bytes of each block that gets sent to a given stripe before rotating to the next stripe (default 102400) + --xtrabackup_stripes uint If greater than 0, use data striping across this many destination files to parallelize data transfer and decompression + --xtrabackup_user string User that xtrabackup will use to connect to the database server. This user must have all necessary privileges. For details, please refer to xtrabackup documentation. diff --git a/go/flags/endtoend/vtctlclient.txt b/go/flags/endtoend/vtctlclient.txt index 207f31905f2..7f7779f0433 100644 --- a/go/flags/endtoend/vtctlclient.txt +++ b/go/flags/endtoend/vtctlclient.txt @@ -1,41 +1,47 @@ Usage of vtctlclient: - --action_timeout duration timeout for the total command (default 1h0m0s) - --alsologtostderr log to standard error as well as files - --datadog-agent-host string host to send spans to. if empty, no tracing will be done - --datadog-agent-port string port to send spans to. if empty, no tracing will be done - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --jaeger-agent-host string host and port to send spans to. if empty, no tracing will be done - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --server string server to use for connection - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --tracer string tracing service to use (default "noop") - --tracing-enable-logging whether to enable logging in the tracing service - --tracing-sampling-rate float sampling rate for the probabilistic jaeger sampler (default 0.1) - --tracing-sampling-type string sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote' (default "const") - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --vtctl_client_protocol string Protocol to use to talk to the vtctl server. (default "grpc") - --vtctld_grpc_ca string the server ca to use to validate servers when connecting - --vtctld_grpc_cert string the cert to use to connect - --vtctld_grpc_crl string the server crl to use to validate server certificates when connecting - --vtctld_grpc_key string the key to use to connect - --vtctld_grpc_server_name string the server name to use to validate server certificate + --action_timeout duration timeout for the total command (default 1h0m0s) + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --datadog-agent-host string host to send spans to. if empty, no tracing will be done + --datadog-agent-port string port to send spans to. if empty, no tracing will be done + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --jaeger-agent-host string host and port to send spans to. if empty, no tracing will be done + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --server string server to use for connection + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --tracer string tracing service to use (default "noop") + --tracing-enable-logging whether to enable logging in the tracing service + --tracing-sampling-rate float sampling rate for the probabilistic jaeger sampler (default 0.1) + --tracing-sampling-type string sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote' (default "const") + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vtctl_client_protocol string Protocol to use to talk to the vtctl server. (default "grpc") + --vtctld_grpc_ca string the server ca to use to validate servers when connecting + --vtctld_grpc_cert string the cert to use to connect + --vtctld_grpc_crl string the server crl to use to validate server certificates when connecting + --vtctld_grpc_key string the key to use to connect + --vtctld_grpc_server_name string the server name to use to validate server certificate diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index eb8c0c552dc..0e471a78ed1 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -18,6 +18,12 @@ Usage of vtctld: --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --cell string cell to use --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. --datadog-agent-host string host to send spans to. if empty, no tracing will be done --datadog-agent-port string port to send spans to. if empty, no tracing will be done diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index a70067a9bc8..08d75f92dca 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -1,37 +1,43 @@ Usage of vtexplain: - --alsologtostderr log to standard error as well as files - --batch-interval duration Interval between logical time slots. (default 10ms) - --dbname string Optional database target to override normal routing - --default_tablet_type topodatapb.TabletType The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY) - --execution-mode string The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc (default "multi") - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --ks-shard-map string JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace - --ks-shard-map-file string File containing json blob of keyspace name -> shard name -> ShardReference object - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --normalize Whether to enable vtgate normalization - --output-mode string Output in human-friendly text or json (default "text") - --planner-version string Sets the query planner version to use when generating the explain output. Valid values are V3 and Gen4. An empty value will use VTGate's default planner - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --replication-mode string The replication mode to simulate -- must be set to either ROW or STATEMENT (default "ROW") - --schema string The SQL table schema - --schema-file string Identifies the file that contains the SQL table schema - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --shards int Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored. (default 2) - --sql string A list of semicolon-delimited SQL commands to analyze - --sql-file string Identifies the file that contains the SQL commands to analyze - --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) - --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --vschema string Identifies the VTGate routing schema - --vschema-file string Identifies the VTGate routing schema file + --alsologtostderr log to standard error as well as files + --batch-interval duration Interval between logical time slots. (default 10ms) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --dbname string Optional database target to override normal routing + --default_tablet_type topodatapb.TabletType The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY) + --execution-mode string The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc (default "multi") + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --ks-shard-map string JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace + --ks-shard-map-file string File containing json blob of keyspace name -> shard name -> ShardReference object + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --normalize Whether to enable vtgate normalization + --output-mode string Output in human-friendly text or json (default "text") + --planner-version string Sets the query planner version to use when generating the explain output. Valid values are V3 and Gen4. An empty value will use VTGate's default planner + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --replication-mode string The replication mode to simulate -- must be set to either ROW or STATEMENT (default "ROW") + --schema string The SQL table schema + --schema-file string Identifies the file that contains the SQL table schema + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --shards int Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored. (default 2) + --sql string A list of semicolon-delimited SQL commands to analyze + --sql-file string Identifies the file that contains the SQL commands to analyze + --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) + --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vschema string Identifies the VTGate routing schema + --vschema-file string Identifies the VTGate routing schema file diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 842aa288779..46649d7cfce 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -1,3 +1,4 @@ +ERROR: logging before flag.Parse: I0516 13:37:57.933157 74801 replicationlag.go:84] discovery.low_replication_lag: 30s 0s Usage of vtgate: --allowed_tablet_types strings Specifies the tablet types this vtgate is allowed to route queries to. Should be provided as a comma-separated set of tablet types. --alsologtostderr log to standard error as well as files @@ -11,6 +12,12 @@ Usage of vtgate: --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --cell string cell to use --cells_to_watch string comma-separated list of cells for watching tablets + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. --datadog-agent-host string host to send spans to. if empty, no tracing will be done --datadog-agent-port string port to send spans to. if empty, no tracing will be done diff --git a/go/flags/endtoend/vtgr.txt b/go/flags/endtoend/vtgr.txt index b70b84d21f3..17501677154 100644 --- a/go/flags/endtoend/vtgr.txt +++ b/go/flags/endtoend/vtgr.txt @@ -1,84 +1,90 @@ Usage of vtgr: - --abort_rebootstrap Don't allow vtgr to rebootstrap an existing group. - --alsologtostderr log to standard error as well as files - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" - --consul_auth_static_file string JSON File to read the topos/tokens from. - --db-credentials-file string db credentials file; send SIGHUP to reload this file - --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") - --db-credentials-vault-addr string URL to Vault server - --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds - --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") - --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable - --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable - --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) - --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate - --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable - --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) - --db_config string Full path to db config file that will be used by VTGR. - --db_flavor string MySQL flavor override. (default "MySQL56") - --db_port int Local mysql port, set this to enable local fast check. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --enable_heartbeat_check Enable heartbeat checking, set together with --group_heartbeat_threshold. - --gr_port int Port to bootstrap a MySQL group. (default 33061) - --group_heartbeat_threshold int VTGR will trigger backoff on inconsistent state if the group heartbeat staleness exceeds this threshold (in seconds). Should be used along with --enable_heartbeat_check. - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --ping_tablet_timeout duration time to wait when we ping a tablet (default 2s) - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --refresh_interval duration Refresh interval to load tablets. (default 10s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --scan_interval duration Scan interval to diagnose and repair. (default 3s) - --scan_repair_timeout duration Time to wait for a Diagnose and repair operation. (default 3s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --vtgr_config string Config file for vtgr. + --abort_rebootstrap Don't allow vtgr to rebootstrap an existing group. + --alsologtostderr log to standard error as well as files + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --consul_auth_static_file string JSON File to read the topos/tokens from. + --db-credentials-file string db credentials file; send SIGHUP to reload this file + --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") + --db-credentials-vault-addr string URL to Vault server + --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds + --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") + --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable + --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable + --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) + --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate + --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable + --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) + --db_config string Full path to db config file that will be used by VTGR. + --db_flavor string MySQL flavor override. (default "MySQL56") + --db_port int Local mysql port, set this to enable local fast check. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --enable_heartbeat_check Enable heartbeat checking, set together with --group_heartbeat_threshold. + --gr_port int Port to bootstrap a MySQL group. (default 33061) + --group_heartbeat_threshold int VTGR will trigger backoff on inconsistent state if the group heartbeat staleness exceeds this threshold (in seconds). Should be used along with --enable_heartbeat_check. + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --ping_tablet_timeout duration time to wait when we ping a tablet (default 2s) + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --refresh_interval duration Refresh interval to load tablets. (default 10s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --scan_interval duration Scan interval to diagnose and repair. (default 3s) + --scan_repair_timeout duration Time to wait for a Diagnose and repair operation. (default 3s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vtgr_config string Config file for vtgr. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index 60af343eca9..34960ac297f 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -1,87 +1,93 @@ Usage of vtorc: - --alsologtostderr log to standard error as well as files - --audit-file-location string File location where the audit logs are to be stored - --audit-purge-duration duration Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s) - --audit-to-backend Whether to store the audit log in the VTOrc database - --audit-to-syslog Whether to store the audit log in the syslog - --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" - --config string config file name - --consul_auth_static_file string JSON File to read the topos/tokens from. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --instance-poll-time duration Timer duration on which VTOrc refreshes MySQL information (default 5s) - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --max-stack-size int configure the maximum stack size in bytes (default 67108864) - --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) - --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) - --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. - --port int port for the server - --pprof strings enable profiling - --prevent-cross-cell-failover Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --reasonable-replication-lag duration Maximum replication lag on replicas which is deemed to be acceptable (default 10s) - --recovery-period-block-duration duration Duration for which a new recovery is blocked on an instance after running a recovery (default 30s) - --recovery-poll-duration duration Timer duration on which VTOrc polls its database to run a recovery (default 1s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --shutdown_wait_time duration Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s) - --snapshot-topology-interval duration Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours - --sqlite-data-file string SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared") - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_k8s_context string The kubeconfig context to use, overrides the 'current-context' from the config - --topo_k8s_kubeconfig string Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod - --topo_k8s_namespace string The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) + --alsologtostderr log to standard error as well as files + --audit-file-location string File location where the audit logs are to be stored + --audit-purge-duration duration Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s) + --audit-to-backend Whether to store the audit log in the VTOrc database + --audit-to-syslog Whether to store the audit log in the syslog + --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --config string config file name + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --consul_auth_static_file string JSON File to read the topos/tokens from. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --instance-poll-time duration Timer duration on which VTOrc refreshes MySQL information (default 5s) + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) + --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) + --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. + --port int port for the server + --pprof strings enable profiling + --prevent-cross-cell-failover Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --reasonable-replication-lag duration Maximum replication lag on replicas which is deemed to be acceptable (default 10s) + --recovery-period-block-duration duration Duration for which a new recovery is blocked on an instance after running a recovery (default 30s) + --recovery-poll-duration duration Timer duration on which VTOrc polls its database to run a recovery (default 1s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --shutdown_wait_time duration Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s) + --snapshot-topology-interval duration Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours + --sqlite-data-file string SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared") + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_k8s_context string The kubeconfig context to use, overrides the 'current-context' from the config + --topo_k8s_kubeconfig string Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod + --topo_k8s_namespace string The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index a0577bad142..d7f415bc360 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -34,6 +34,12 @@ Usage of vttablet: --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") --compression-engine-name string compressor engine used for compression. (default "pargzip") --compression-level int what level to pass to the compressor. (default 1) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) --consolidator-stream-total-size int Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728) --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index be553107af0..54c199be756 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -15,6 +15,12 @@ Usage of vttestserver: --charset string MySQL charset (default "utf8mb4") --compression-engine-name string compressor engine used for compression. (default "pargzip") --compression-level int what level to pass to the compressor. (default 1) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. --data_dir string Directory where the data files will be placed, defaults to a random directory under /vt/vtdataroot --dba_idle_timeout duration Idle timeout for dba connections (default 1m0s) diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index e7e41c4cb4d..f687e9be3b3 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -1,18 +1,24 @@ Usage of zkctl: - --alsologtostderr log to standard error as well as files - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") - --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") + --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index 6ec026be814..0adb1b5cd8b 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -1,19 +1,25 @@ Usage of zkctld: - --alsologtostderr log to standard error as well as files - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") - --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") + --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname From 860c8655e7f19f16b5033c55773590a957813104 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 14:39:33 -0400 Subject: [PATCH 077/100] dumb dumb Signed-off-by: Andrew Mason --- go/flags/endtoend/mysqlctl.txt | 2 +- go/flags/endtoend/mysqlctld.txt | 2 +- go/flags/endtoend/vtaclcheck.txt | 2 +- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtctlclient.txt | 2 +- go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vtexplain.txt | 2 +- go/flags/endtoend/vtgate.txt | 3 +-- go/flags/endtoend/vtgr.txt | 2 +- go/flags/endtoend/vtorc.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/flags/endtoend/zkctl.txt | 2 +- go/flags/endtoend/zkctld.txt | 2 +- 14 files changed, 14 insertions(+), 15 deletions(-) diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index 4910fc97e2a..38ed1302ceb 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -18,7 +18,7 @@ Global flags: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index 250cbf539d4..6f3622122a8 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -6,7 +6,7 @@ Usage of mysqlctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file diff --git a/go/flags/endtoend/vtaclcheck.txt b/go/flags/endtoend/vtaclcheck.txt index b25e22af0d9..6c92d4bf50d 100644 --- a/go/flags/endtoend/vtaclcheck.txt +++ b/go/flags/endtoend/vtaclcheck.txt @@ -4,7 +4,7 @@ Usage of vtaclcheck: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 4a4b05b251b..8d517e5db63 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -22,7 +22,7 @@ Usage of vtbackup: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtctlclient.txt b/go/flags/endtoend/vtctlclient.txt index 7f7779f0433..62cc29ace38 100644 --- a/go/flags/endtoend/vtctlclient.txt +++ b/go/flags/endtoend/vtctlclient.txt @@ -4,7 +4,7 @@ Usage of vtctlclient: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --datadog-agent-host string host to send spans to. if empty, no tracing will be done diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 0e471a78ed1..9ec2d622897 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -21,7 +21,7 @@ Usage of vtctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index 08d75f92dca..a5e7c709fe9 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -4,7 +4,7 @@ Usage of vtexplain: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --dbname string Optional database target to override normal routing diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 46649d7cfce..a31e8eeab8b 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -1,4 +1,3 @@ -ERROR: logging before flag.Parse: I0516 13:37:57.933157 74801 replicationlag.go:84] discovery.low_replication_lag: 30s 0s Usage of vtgate: --allowed_tablet_types strings Specifies the tablet types this vtgate is allowed to route queries to. Should be provided as a comma-separated set of tablet types. --alsologtostderr log to standard error as well as files @@ -15,7 +14,7 @@ Usage of vtgate: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtgr.txt b/go/flags/endtoend/vtgr.txt index 17501677154..351773c24f2 100644 --- a/go/flags/endtoend/vtgr.txt +++ b/go/flags/endtoend/vtgr.txt @@ -5,7 +5,7 @@ Usage of vtgr: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index 34960ac297f..e7884faf705 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -10,7 +10,7 @@ Usage of vtorc: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index d7f415bc360..5a134a0aa23 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -37,7 +37,7 @@ Usage of vttablet: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 54c199be756..2eb276374f9 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -18,7 +18,7 @@ Usage of vttestserver: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index f687e9be3b3..bc5df28b00b 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -3,7 +3,7 @@ Usage of zkctl: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index 0adb1b5cd8b..fed6c9ef4f6 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -3,7 +3,7 @@ Usage of zkctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [/Users/andrew/dev/vitess]) + --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit From c1a77da71901e914c28e02cdb7e1551cb956ca43 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 15:08:36 -0400 Subject: [PATCH 078/100] templatize (which requires further escaping) Signed-off-by: Andrew Mason --- go/flags/endtoend/flags_test.go | 20 ++++++++++++++++++-- go/flags/endtoend/mysqlctl.txt | 2 +- go/flags/endtoend/mysqlctld.txt | 2 +- go/flags/endtoend/vtaclcheck.txt | 2 +- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtctlclient.txt | 2 +- go/flags/endtoend/vtctld.txt | 4 ++-- go/flags/endtoend/vtexplain.txt | 2 +- go/flags/endtoend/vtgate.txt | 4 ++-- go/flags/endtoend/vtgr.txt | 2 +- go/flags/endtoend/vtorc.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/flags/endtoend/zkctl.txt | 2 +- go/flags/endtoend/zkctld.txt | 2 +- 15 files changed, 34 insertions(+), 18 deletions(-) diff --git a/go/flags/endtoend/flags_test.go b/go/flags/endtoend/flags_test.go index 1ad69c7fa84..8199c52d1b4 100644 --- a/go/flags/endtoend/flags_test.go +++ b/go/flags/endtoend/flags_test.go @@ -23,8 +23,10 @@ package flags import ( "bytes" _ "embed" + "os" "os/exec" "testing" + "text/template" "vitess.io/vitess/go/test/utils" @@ -105,16 +107,30 @@ var ( ) func TestHelpOutput(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + args := []string{"--help"} for binary, helptext := range helpOutput { t.Run(binary, func(t *testing.T) { + tmpl, err := template.New(binary).Parse(helptext) + require.NoError(t, err) + + var buf bytes.Buffer + err = tmpl.Execute(&buf, struct { + Workdir string + }{ + Workdir: wd, + }) + require.NoError(t, err) + cmd := exec.Command(binary, args...) output := bytes.Buffer{} cmd.Stderr = &output cmd.Stdout = &output - err := cmd.Run() + err = cmd.Run() require.NoError(t, err) - utils.MustMatch(t, helptext, output.String()) + utils.MustMatch(t, buf.String(), output.String()) }) } } diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index 38ed1302ceb..d08e12f66ad 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -18,7 +18,7 @@ Global flags: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index 6f3622122a8..9eb73cf4c62 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -6,7 +6,7 @@ Usage of mysqlctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file diff --git a/go/flags/endtoend/vtaclcheck.txt b/go/flags/endtoend/vtaclcheck.txt index 6c92d4bf50d..866f500c2d1 100644 --- a/go/flags/endtoend/vtaclcheck.txt +++ b/go/flags/endtoend/vtaclcheck.txt @@ -4,7 +4,7 @@ Usage of vtaclcheck: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 8d517e5db63..4d1e5186c74 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -22,7 +22,7 @@ Usage of vtbackup: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtctlclient.txt b/go/flags/endtoend/vtctlclient.txt index 62cc29ace38..c7c2f8c9c2e 100644 --- a/go/flags/endtoend/vtctlclient.txt +++ b/go/flags/endtoend/vtctlclient.txt @@ -4,7 +4,7 @@ Usage of vtctlclient: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --datadog-agent-host string host to send spans to. if empty, no tracing will be done diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 9ec2d622897..ba9c795577a 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -21,7 +21,7 @@ Usage of vtctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. @@ -121,7 +121,7 @@ Usage of vtctld: --tablet_protocol string Protocol to use to make queryservice RPCs to vttablets. (default "grpc") --tablet_refresh_interval duration Tablet refresh interval. (default 1m0s) --tablet_refresh_known_tablets Whether to reload the tablet's address/port map from topo in case they change. (default true) - --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{.GetTabletHostPort}}") + --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}") --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index a5e7c709fe9..42c510a1ccc 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -4,7 +4,7 @@ Usage of vtexplain: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --dbname string Optional database target to override normal routing diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index a31e8eeab8b..96504b787ce 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -14,7 +14,7 @@ Usage of vtgate: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. @@ -174,7 +174,7 @@ Usage of vtgate: --tablet_refresh_interval duration Tablet refresh interval. (default 1m0s) --tablet_refresh_known_tablets Whether to reload the tablet's address/port map from topo in case they change. (default true) --tablet_types_to_wait strings Wait till connected for specified tablet types during Gateway initialization. Should be provided as a comma-separated set of tablet types. - --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{.GetTabletHostPort}}") + --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}") --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. diff --git a/go/flags/endtoend/vtgr.txt b/go/flags/endtoend/vtgr.txt index 351773c24f2..128bf07a0dd 100644 --- a/go/flags/endtoend/vtgr.txt +++ b/go/flags/endtoend/vtgr.txt @@ -5,7 +5,7 @@ Usage of vtgr: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index e7884faf705..c177fb871ab 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -10,7 +10,7 @@ Usage of vtorc: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 5a134a0aa23..8e88a7c6fd0 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -37,7 +37,7 @@ Usage of vttablet: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 2eb276374f9..f3bc4da067e 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -18,7 +18,7 @@ Usage of vttestserver: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index bc5df28b00b..a26e4e14507 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -3,7 +3,7 @@ Usage of zkctl: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index fed6c9ef4f6..c8d5526e9d5 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -3,7 +3,7 @@ Usage of zkctld: --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") - --config-path strings Paths to search for config files in. (default [home/runner/work/vitess/vitess/go/flags/endtoend]) + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) --config-type string Config file type (omit to infer config type from file extension). -h, --help display usage and exit From d4bd4b590eff1777b4722396238f16670eb244aa Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 16 May 2023 16:54:01 -0400 Subject: [PATCH 079/100] lint Signed-off-by: Andrew Mason --- go/viperutil/funcs/get.go | 4 +--- go/viperutil/vipertest/stub.go | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go/viperutil/funcs/get.go b/go/viperutil/funcs/get.go index 198ab11edcf..99022de9504 100644 --- a/go/viperutil/funcs/get.go +++ b/go/viperutil/funcs/get.go @@ -28,9 +28,7 @@ func GetPath(v *viper.Viper) func(key string) []string { return func(key string) (paths []string) { for _, val := range v.GetStringSlice(key) { if val != "" { - for _, path := range strings.Split(val, ":") { - paths = append(paths, path) - } + paths = append(paths, strings.Split(val, ":")...) } } diff --git a/go/viperutil/vipertest/stub.go b/go/viperutil/vipertest/stub.go index 700d3639566..e4673779454 100644 --- a/go/viperutil/vipertest/stub.go +++ b/go/viperutil/vipertest/stub.go @@ -33,10 +33,9 @@ import ( // // It fails the test if a caller attempts to stub the same value multiple times // to a particular viper. -func Stub[T any](t *testing.T, v *viper.Viper, val viperutil.Value[T]) (reset func()) { +func Stub[T any](t *testing.T, v *viper.Viper, val viperutil.Value[T]) func() { t.Helper() - reset = func() {} if !assert.False(t, v.InConfig(val.Key()), "value for key %s already stubbed", val.Key()) { return func() {} } From cff1efa4cce69771adf80751b69cd1b805fe528b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 17 May 2023 10:14:40 -0400 Subject: [PATCH 080/100] docs Signed-off-by: Andrew Mason --- doc/viper/viper.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index 1ba7f7b0992..6c9b076f323 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -245,6 +245,23 @@ For more information on how viper searches for config files, see the [documentat If viper was able to locate and load a config file, `LoadConfig` will then configure the dynamic registry to set up a watch on that file, enabling all dynamic values to pick up changes to that file for the remainder of the program's execution. If no config file was used, then dynamic values behave exactly like static values (i.e. the dynamic registry copies in the settings loaded into the static registry, but does not set up a file watch). +### Re-persistence for Dynamic Values + +Prior to the introduction of viper in Vitess, certain components (such as `vttablet` or `vtgate`) exposed `/debug/env` HTTP endpoints that permitted the user to modify certain configuration parameters at runtime. + +This behavior is still supported, and to maintain consistency between update mechanisms, if: +- A config file was loaded at startup +- A value is configured with the `Dynamic: true` option + +then in-memory updates to that value (via `.Set()`) will be written back to disk. +If we skipped this step, then the next time viper reloaded the disk config, the in-memory change would be undone, since viper does a full load rather than something more differential. +Unfortunately, this seems unavoidable. + +To migitate against potentially writing to disk "too often" for a given user, the `--config-persistence-min-interval` flag defines the _minimum_ time to wait between writes. +Internally, the system is notified to write "soon" only when a dynamic value is updated. +If the wait period has elapsed between changes, a write happens immediately; otherwise, the system waits out the remainder of the period and persists any changes that happened while it was waiting. +Setting this interval to zero means that writes happen immediately. + ## Auto-Documentation One of the benefits of all values being created through a single function is that we can pretty easily build tooling to generate documentation for the config values available to a given binary. @@ -268,6 +285,13 @@ The exact formatting can be tweaked, obviously, but as an example, something lik If/when we migrate other binaries to cobra, we can figure out how to combine this documntation with cobra's doc-generation tooling (which we use for `vtctldclient` and `vtadmin`). +## Debug Endpoint + +Any component that parses its flags via one of `servenv`'s parsing methods will get an HTTP endpoint registered at `/debug/config` which displays the full viper configuration for debugging purposes. +It accepts a query parameter to control the format; anything in `viper.SupportedExts` is permitted. + +Components that do not use `servenv` to parse their flags may manually register the `(go/viperutil/debug).HandlerFunc` if they wish. + ## Caveats and Gotchas - Config keys are case-insensitive. From fbf29899ac851b95a86d526b7788918a2cbf6d7c Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 19 May 2023 06:46:54 -0400 Subject: [PATCH 081/100] Update doc/viper/viper.md Co-authored-by: Deepthi Sigireddi Signed-off-by: Andrew Mason --- doc/viper/viper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index 6c9b076f323..07002bbed1f 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -139,7 +139,7 @@ The full suite of types, both supported and panic-inducing, are documented by wa ### Dynamic values Values can be configured to be either static or dynamic. -Static values are loaded once at startup (more precisely, when `viperutil.LoadConfig` is called), and whatever value is loaded at the point will be the result of calling `Get` on that value for the remainder of the processes lifetime. +Static values are loaded once at startup (more precisely, when `viperutil.LoadConfig` is called), and whatever value is loaded at the point will be the result of calling `Get` on that value for the remainder of the process's lifetime. Dynamic values, conversely, may respond to config changes. In order for dynamic configs to be truly dynamic, `LoadConfig` must have found a config file (as opposed to pulling values entirely from defaults, flags, and environment variables). From dae6669391e3a9276f8f95b64b480e2f8176a8d1 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 19 May 2023 06:48:17 -0400 Subject: [PATCH 082/100] back to "warn on default", update tests to handle it Signed-off-by: Andrew Mason --- go/test/endtoend/cluster/cluster_util.go | 4 ++++ go/test/endtoend/cluster/vtctlclient_process.go | 2 +- go/viperutil/config.go | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 59327ff6503..4a48626ed28 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -185,6 +185,10 @@ func filterResultForWarning(input string) string { if strings.Contains(line, "WARNING: vtctl should only be used for VDiff v1 workflows. Please use VDiff v2 and consider using vtctldclient for all other commands.") { continue } + + if strings.Contains(line, "Failed to read in config") && strings.Contains(line, `Config File "vtconfig" Not Found in`) { + continue + } result = result + line + "\n" } return result diff --git a/go/test/endtoend/cluster/vtctlclient_process.go b/go/test/endtoend/cluster/vtctlclient_process.go index 3c4941da2c2..cf75ec1a5fa 100644 --- a/go/test/endtoend/cluster/vtctlclient_process.go +++ b/go/test/endtoend/cluster/vtctlclient_process.go @@ -215,7 +215,7 @@ func (vtctlclient *VtctlClientProcess) ExecuteCommandWithOutput(args ...string) } time.Sleep(retryDelay) } - return filterResultWhenRunsForCoverage(resultStr), err + return filterResultForWarning(filterResultWhenRunsForCoverage(resultStr)), err } // VtctlClientProcessInstance returns a VtctlProcess handle for vtctlclient process diff --git a/go/viperutil/config.go b/go/viperutil/config.go index b1a279c14a2..347dc6cfa0e 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -70,7 +70,7 @@ var ( configFileNotFoundHandling = Configure( "config.notfound.handling", Options[ConfigFileNotFoundHandling]{ - Default: IgnoreConfigFileNotFound, + Default: WarnOnConfigFileNotFound, GetFunc: getHandlingValue, }, ) From b76a4fe8bdd00d4303afad84e0b8fc5532c6a72d Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 19 May 2023 06:50:43 -0400 Subject: [PATCH 083/100] flag testdata Signed-off-by: Andrew Mason --- go/flags/endtoend/mysqlctl.txt | 2 +- go/flags/endtoend/mysqlctld.txt | 2 +- go/flags/endtoend/vtaclcheck.txt | 2 +- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtctlclient.txt | 2 +- go/flags/endtoend/vtctld.txt | 2 +- go/flags/endtoend/vtexplain.txt | 2 +- go/flags/endtoend/vtgate.txt | 2 +- go/flags/endtoend/vtgr.txt | 2 +- go/flags/endtoend/vtorc.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/flags/endtoend/zkctl.txt | 2 +- go/flags/endtoend/zkctld.txt | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index d08e12f66ad..2472528260b 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -16,7 +16,7 @@ Global flags: --app_pool_size int Size of the connection pool for app connections (default 40) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index 9eb73cf4c62..8b0ca101cb4 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -4,7 +4,7 @@ Usage of mysqlctld: --app_pool_size int Size of the connection pool for app connections (default 40) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtaclcheck.txt b/go/flags/endtoend/vtaclcheck.txt index 866f500c2d1..d28eb7351b9 100644 --- a/go/flags/endtoend/vtaclcheck.txt +++ b/go/flags/endtoend/vtaclcheck.txt @@ -2,7 +2,7 @@ Usage of vtaclcheck: --acl-file string The path of the JSON ACL file to check --alsologtostderr log to standard error as well as files --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 4d1e5186c74..f825e764e6c 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -20,7 +20,7 @@ Usage of vtbackup: --compression-level int what level to pass to the compressor. (default 1) --concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtctlclient.txt b/go/flags/endtoend/vtctlclient.txt index c7c2f8c9c2e..cbb6d74cb9d 100644 --- a/go/flags/endtoend/vtctlclient.txt +++ b/go/flags/endtoend/vtctlclient.txt @@ -2,7 +2,7 @@ Usage of vtctlclient: --action_timeout duration timeout for the total command (default 1h0m0s) --alsologtostderr log to standard error as well as files --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index ba9c795577a..7bc57f53630 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -19,7 +19,7 @@ Usage of vtctld: --cell string cell to use --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index 42c510a1ccc..29ffb35f343 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -2,7 +2,7 @@ Usage of vtexplain: --alsologtostderr log to standard error as well as files --batch-interval duration Interval between logical time slots. (default 10ms) --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 96504b787ce..bb576961c29 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -12,7 +12,7 @@ Usage of vtgate: --cell string cell to use --cells_to_watch string comma-separated list of cells for watching tablets --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtgr.txt b/go/flags/endtoend/vtgr.txt index 128bf07a0dd..c553ee8015c 100644 --- a/go/flags/endtoend/vtgr.txt +++ b/go/flags/endtoend/vtgr.txt @@ -3,7 +3,7 @@ Usage of vtgr: --alsologtostderr log to standard error as well as files --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index c177fb871ab..b38cca7374b 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -8,7 +8,7 @@ Usage of vtorc: --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" --config string config file name --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 8e88a7c6fd0..d8956aea341 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -35,7 +35,7 @@ Usage of vttablet: --compression-engine-name string compressor engine used for compression. (default "pargzip") --compression-level int what level to pass to the compressor. (default 1) --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index f3bc4da067e..4254f58c398 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -16,7 +16,7 @@ Usage of vttestserver: --compression-engine-name string compressor engine used for compression. (default "pargzip") --compression-level int what level to pass to the compressor. (default 1) --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index a26e4e14507..f52a5f7fe1b 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -1,7 +1,7 @@ Usage of zkctl: --alsologtostderr log to standard error as well as files --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index c8d5526e9d5..e2187ead03f 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -1,7 +1,7 @@ Usage of zkctld: --alsologtostderr log to standard error as well as files --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. - --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default ignore) + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) --config-name string Name of the config file (without extension) to search for. (default "vtconfig") --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) From 485d09fc9860f8f9863a50d2f822b0f91d7d884c Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Fri, 19 May 2023 14:23:52 -0400 Subject: [PATCH 084/100] fix filtering quirk Signed-off-by: Andrew Mason --- go/test/endtoend/cluster/cluster_util.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 4a48626ed28..33d46aa1da6 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -191,6 +191,13 @@ func filterResultForWarning(input string) string { } result = result + line + "\n" } + + if result == strings.Repeat("\n", len(result)) { + // If every line was filtered, return an empty string rather than a + // bunch of newline characters. + return "" + } + return result } From a00b9a4ca8a6b7e24f21dc5536f116fb80b35606 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sat, 20 May 2023 10:06:25 -0400 Subject: [PATCH 085/100] better newline handling to avoid double-newline at end Signed-off-by: Andrew Mason --- go/test/endtoend/cluster/cluster_util.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 33d46aa1da6..07a9e09df45 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -181,7 +181,7 @@ func getTablet(tabletGrpcPort int, hostname string) *topodatapb.Tablet { func filterResultForWarning(input string) string { lines := strings.Split(input, "\n") var result string - for _, line := range lines { + for i, line := range lines { if strings.Contains(line, "WARNING: vtctl should only be used for VDiff v1 workflows. Please use VDiff v2 and consider using vtctldclient for all other commands.") { continue } @@ -189,13 +189,12 @@ func filterResultForWarning(input string) string { if strings.Contains(line, "Failed to read in config") && strings.Contains(line, `Config File "vtconfig" Not Found in`) { continue } - result = result + line + "\n" - } - if result == strings.Repeat("\n", len(result)) { - // If every line was filtered, return an empty string rather than a - // bunch of newline characters. - return "" + result += line + + if i < len(lines)-1 { + result += "\n" + } } return result From 67cd23d4c1a0db355380d7f87d323eaee916d73f Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Sun, 21 May 2023 13:44:53 -0400 Subject: [PATCH 086/100] refactor to colocate persistence with the sync viper Signed-off-by: Andrew Mason --- go/viperutil/config.go | 13 +-- go/viperutil/internal/config/config.go | 122 ------------------------ go/viperutil/internal/sync/sync.go | 97 +++++++++++++++++-- go/viperutil/internal/sync/sync_test.go | 107 ++++++++++++++++++++- go/viperutil/internal/value/value.go | 2 - 5 files changed, 195 insertions(+), 146 deletions(-) delete mode 100644 go/viperutil/internal/config/config.go diff --git a/go/viperutil/config.go b/go/viperutil/config.go index 347dc6cfa0e..a551bbcf07c 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -30,7 +30,6 @@ import ( "github.com/spf13/viper" "vitess.io/vitess/go/viperutil/funcs" - "vitess.io/vitess/go/viperutil/internal/config" "vitess.io/vitess/go/viperutil/internal/log" "vitess.io/vitess/go/viperutil/internal/registry" "vitess.io/vitess/go/viperutil/internal/value" @@ -161,7 +160,6 @@ func LoadConfig() (context.CancelFunc, error) { err = registry.Static.ReadInConfig() } - usingConfigFile := true if err != nil { if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { msg := "Failed to read in config %s: %s" @@ -171,7 +169,6 @@ func LoadConfig() (context.CancelFunc, error) { fallthrough // after warning, ignore the error case IgnoreConfigFileNotFound: err = nil - usingConfigFile = false case ErrorOnConfigFileNotFound: log.ERROR(msg, registry.Static.ConfigFileUsed(), nferr.Error()) case ExitOnConfigFileNotFound: @@ -184,15 +181,7 @@ func LoadConfig() (context.CancelFunc, error) { return nil, err } - if err := registry.Dynamic.Watch(registry.Static); err != nil { - return nil, err - } - - if usingConfigFile { - return config.PersistChanges(context.Background(), configPersistenceMinInterval.Get()), nil - } - - return func() {}, nil + return registry.Dynamic.Watch(context.Background(), registry.Static, configPersistenceMinInterval.Get()) } // NotifyConfigReload adds a subscription that the dynamic registry will attempt diff --git a/go/viperutil/internal/config/config.go b/go/viperutil/internal/config/config.go deleted file mode 100644 index 4cf868c7a71..00000000000 --- a/go/viperutil/internal/config/config.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2023 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package config - -import ( - "context" - "time" - - "vitess.io/vitess/go/viperutil/internal/log" - "vitess.io/vitess/go/viperutil/internal/registry" -) - -var ( - ch chan struct{} -) - -// PersistChanges starts a background goroutine to persist changes made to the -// dynamic registry in-memory (i.e. the "live" config) back to disk (i.e. the -// "disk" config) are persisted back to disk. It returns a cancel func to stop -// the persist loop, which the caller is responsible for calling during -// shutdown (see package servenv for an example). -// -// This does two things — one which is a nice-to-have, and another which is -// necessary for correctness. -// -// 1. Writing in-memory changes (which usually occur through a request to a -// /debug/env endpoint) ensures they are persisted across process restarts. -// 2. Writing in-memory changes ensures that subsequent modifications to the -// config file do not clobber those changes. Because viper loads the entire -// config on-change, rather than an incremental (diff) load, if a user were to -// edit an unrelated key (keyA) in the file, and we did not persist the -// in-memory change (keyB), then future calls to keyB.Get() would return the -// older value. -func PersistChanges(ctx context.Context, minWaitInterval time.Duration) context.CancelFunc { - if ch != nil { - panic("PersistChanges already called") - } - - ch = make(chan struct{}, 1) - - var timer *time.Timer - if minWaitInterval > 0 { - timer = time.NewTimer(minWaitInterval) - } - - persistOnce := func() { - if err := registry.Dynamic.WriteConfig(); err != nil { - log.ERROR("failed to persist config changes back to disk: %s", err.Error()) - // If we failed to persist, don't wait the entire interval before - // writing again, instead writing immediately on the next request. - if timer != nil { - if !timer.Stop() { - <-timer.C - } - - timer = nil - } - } - - switch { - case minWaitInterval == 0: - return - case timer == nil: - timer = time.NewTimer(minWaitInterval) - default: - timer.Reset(minWaitInterval) - } - } - - ctx, cancel := context.WithCancel(ctx) - go func() { - defer close(ch) - - for { - select { - case <-ctx.Done(): - return - case <-ch: - if timer == nil { - persistOnce() - continue - } - - select { - case <-ctx.Done(): - return - case <-timer.C: - persistOnce() - } - } - } - }() - - return cancel -} - -// NotifyChanged signals to the persist loop started by PersistChanges() that -// something in the config has changed, and should be persisted soon. -func NotifyChanged() { - if ch == nil { - return - } - - select { - case ch <- struct{}{}: - default: - } -} diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index 1aab57d8005..3a405a8ee93 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -17,13 +17,17 @@ limitations under the License. package sync import ( + "context" "errors" "fmt" "sync" + "time" "github.com/fsnotify/fsnotify" "github.com/spf13/pflag" "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/internal/log" ) // Viper is a wrapper around a pair of viper.Viper instances to provide config- @@ -42,14 +46,17 @@ type Viper struct { subscribers []chan<- struct{} watchingConfig bool + + setCh chan struct{} } // New returns a new synced Viper. func New() *Viper { return &Viper{ - disk: viper.New(), - live: viper.New(), - keys: map[string]*sync.RWMutex{}, + disk: viper.New(), + live: viper.New(), + keys: map[string]*sync.RWMutex{}, + setCh: make(chan struct{}, 1), } } @@ -69,6 +76,7 @@ func (v *Viper) Set(key string, value any) { // We must not update v.disk here; explicit calls to Set will supercede all // future config reloads. v.live.Set(key, value) + v.setCh <- struct{}{} } // ErrDuplicateWatch is returned when Watch is called on a synced Viper which @@ -83,25 +91,42 @@ var ErrDuplicateWatch = errors.New("duplicate watch") // If the given static viper did not load a config file (and is instead relying // purely on defaults, flags, and environment variables), then the settings of // that viper are merged over, and this synced Viper may be used to set up an -// actual watch later. +// actual watch later. Additionally, this starts a background goroutine to +// persist changes made in-memory back to disk. It returns a cancel func to stop +// the persist loop, which the caller is responsible for calling during +// shutdown (see package servenv for an example). +// +// This does two things — one which is a nice-to-have, and another which is +// necessary for correctness. +// +// 1. Writing in-memory changes (which usually occur through a request to a +// /debug/env endpoint) ensures they are persisted across process restarts. +// 2. Writing in-memory changes ensures that subsequent modifications to the +// config file do not clobber those changes. Because viper loads the entire +// config on-change, rather than an incremental (diff) load, if a user were to +// edit an unrelated key (keyA) in the file, and we did not persist the +// in-memory change (keyB), then future calls to keyB.Get() would return the +// older value. // // If this synced viper is already watching a config file, this function returns // an ErrDuplicateWatch. Other errors may be returned via underlying viper code // to ensure the config file can be read in properly. -func (v *Viper) Watch(static *viper.Viper) error { +func (v *Viper) Watch(ctx context.Context, static *viper.Viper, minWaitInterval time.Duration) (cancel context.CancelFunc, err error) { if v.watchingConfig { - return fmt.Errorf("%w: viper is already watching %s", ErrDuplicateWatch, v.disk.ConfigFileUsed()) + return nil, fmt.Errorf("%w: viper is already watching %s", ErrDuplicateWatch, v.disk.ConfigFileUsed()) } + ctx, cancel = context.WithCancel(ctx) + cfg := static.ConfigFileUsed() if cfg == "" { // No config file to watch, just merge the settings and return. - return v.live.MergeConfigMap(static.AllSettings()) + return cancel, v.live.MergeConfigMap(static.AllSettings()) } v.disk.SetConfigFile(cfg) if err := v.disk.ReadInConfig(); err != nil { - return err + return nil, err } v.watchingConfig = true @@ -124,7 +149,61 @@ func (v *Viper) Watch(static *viper.Viper) error { }) v.disk.WatchConfig() - return nil + go v.persistChanges(ctx, minWaitInterval) + + return cancel, nil +} + +func (v *Viper) persistChanges(ctx context.Context, minWaitInterval time.Duration) { + defer close(v.setCh) + + var timer *time.Timer + if minWaitInterval > 0 { + timer = time.NewTimer(minWaitInterval) + } + + persistOnce := func() { + if err := v.WriteConfig(); err != nil { + log.ERROR("failed to persist config changes back to disk: %s", err.Error()) + // If we failed to persist, don't wait the entire interval before + // writing again, instead writing immediately on the next request. + if timer != nil { + if !timer.Stop() { + <-timer.C + } + + timer = nil + } + } + + switch { + case minWaitInterval == 0: + return + case timer == nil: + timer = time.NewTimer(minWaitInterval) + default: + timer.Reset(minWaitInterval) + } + } + + for { + select { + case <-ctx.Done(): + return + case <-v.setCh: + if timer == nil { + persistOnce() + continue + } + + select { + case <-ctx.Done(): + return + case <-timer.C: + persistOnce() + } + } + } } // WriteConfig writes the live viper config back to disk. diff --git a/go/viperutil/internal/sync/sync_test.go b/go/viperutil/internal/sync/sync_test.go index d97f8b3d6da..a08d2ae1967 100644 --- a/go/viperutil/internal/sync/sync_test.go +++ b/go/viperutil/internal/sync/sync_test.go @@ -19,14 +19,17 @@ package sync_test import ( "context" "encoding/json" + "fmt" "math/rand" "os" + "strings" "sync" "testing" "time" "github.com/fsnotify/fsnotify" "github.com/spf13/viper" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "vitess.io/vitess/go/viperutil" @@ -34,6 +37,106 @@ import ( "vitess.io/vitess/go/viperutil/internal/value" ) +func TestPersistConfig(t *testing.T) { + type config struct { + Foo int `json:"foo"` + } + + loadConfig := func(t *testing.T, f *os.File) config { + t.Helper() + + data, err := os.ReadFile(f.Name()) + require.NoError(t, err) + + var cfg config + require.NoError(t, json.Unmarshal(data, &cfg)) + + return cfg + } + + setup := func(t *testing.T, v *vipersync.Viper, minWaitInterval time.Duration) (*os.File, chan struct{}) { + tmp, err := os.CreateTemp(t.TempDir(), fmt.Sprintf("%s_*.json", strings.ReplaceAll(t.Name(), "/", "_"))) + require.NoError(t, err) + + t.Cleanup(func() { os.Remove(tmp.Name()) }) + + cfg := config{ + Foo: jitter(1, 100), + } + + data, err := json.Marshal(&cfg) + require.NoError(t, err) + + _, err = tmp.Write(data) + require.NoError(t, err) + + static := viper.New() + static.SetConfigFile(tmp.Name()) + require.NoError(t, static.ReadInConfig()) + + ch := make(chan struct{}, 1) + v.Notify(ch) + + cancel, err := v.Watch(context.Background(), static, minWaitInterval) + require.NoError(t, err) + t.Cleanup(cancel) + + return tmp, ch + } + + t.Run("basic", func(t *testing.T) { + v := vipersync.New() + + minPersistWaitInterval := 2 * time.Second + get := vipersync.AdaptGetter("foo", viperutil.GetFuncForType[int](), v) + f, ch := setup(t, v, minPersistWaitInterval) + + old := get("foo") + loadConfig(t, f) + v.Set("foo", old+1) + // This should happen immediately in-memory and on-disk. + assert.Equal(t, old+1, get("foo")) + <-ch + assert.Equal(t, old+1, loadConfig(t, f).Foo) + + v.Set("foo", old+2) + // This should _also_ happen immediately in-memory, but not on-disk. + // It will take up to 2 * minPersistWaitInterval to reach the disk. + assert.Equal(t, old+2, get("foo")) + assert.Equal(t, old+1, loadConfig(t, f).Foo) + + select { + case <-ch: + case <-time.After(2 * minPersistWaitInterval): + assert.Fail(t, "config was not persisted quickly enough", "config took longer than %s to persist (minPersistWaitInterval = %s)", 2*minPersistWaitInterval, minPersistWaitInterval) + } + + assert.Equal(t, old+2, loadConfig(t, f).Foo) + }) + + t.Run("no persist interval", func(t *testing.T) { + v := vipersync.New() + + var minPersistWaitInterval time.Duration + get := vipersync.AdaptGetter("foo", viperutil.GetFuncForType[int](), v) + f, ch := setup(t, v, minPersistWaitInterval) + + old := get("foo") + loadConfig(t, f) + v.Set("foo", old+1) + // This should happen immediately in-memory and on-disk. + assert.Equal(t, old+1, get("foo")) + <-ch + assert.Equal(t, old+1, loadConfig(t, f).Foo) + + v.Set("foo", old+2) + // This should _also_ happen immediately in-memory, and on-disk. + assert.Equal(t, old+2, get("foo")) + <-ch + assert.Equal(t, old+2, loadConfig(t, f).Foo) + }) +} + func TestWatchConfig(t *testing.T) { type config struct { A, B int @@ -101,7 +204,9 @@ func TestWatchConfig(t *testing.T) { return v.GetInt }, sv) - require.NoError(t, sv.Watch(v)) + cancel, err := sv.Watch(context.Background(), v, 0) + require.NoError(t, err) + defer cancel() var wg sync.WaitGroup ctx, cancel := context.WithCancel(context.Background()) diff --git a/go/viperutil/internal/value/value.go b/go/viperutil/internal/value/value.go index efe60881c4e..fbc754cb993 100644 --- a/go/viperutil/internal/value/value.go +++ b/go/viperutil/internal/value/value.go @@ -23,7 +23,6 @@ import ( "github.com/spf13/pflag" "github.com/spf13/viper" - "vitess.io/vitess/go/viperutil/internal/config" "vitess.io/vitess/go/viperutil/internal/registry" "vitess.io/vitess/go/viperutil/internal/sync" ) @@ -167,5 +166,4 @@ func (val *Dynamic[T]) Registry() registry.Bindable { func (val *Dynamic[T]) Set(v T) { registry.Dynamic.Set(val.KeyName, v) - config.NotifyChanged() } From ad52e4e1d027a1cf96a79fe7846fc081d75c7f9b Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Mon, 22 May 2023 09:37:11 -0400 Subject: [PATCH 087/100] forgot to copy over non-blocking send Signed-off-by: Andrew Mason --- go/viperutil/internal/sync/sync.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index 3a405a8ee93..929e20377b1 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -76,7 +76,10 @@ func (v *Viper) Set(key string, value any) { // We must not update v.disk here; explicit calls to Set will supercede all // future config reloads. v.live.Set(key, value) - v.setCh <- struct{}{} + select { + case v.setCh <- struct{}{}: + default: + } } // ErrDuplicateWatch is returned when Watch is called on a synced Viper which From 2ee8df9f6a344fd26098acfb1fc96a7ad0a8af1a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:21:13 -0400 Subject: [PATCH 088/100] Update doc/viper/viper.md Co-authored-by: Matt Lord Signed-off-by: Andrew Mason --- doc/viper/viper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index 07002bbed1f..659434c40e2 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -10,7 +10,7 @@ It acts as a registry for configuration values coming from a variety of sources, - Environment variables. - Command-line flags, primarily from `pflag.Flag` types. -It is used by a wide variety of Go projects, including [hugo][hugo] and [the kubernetes operator][kops]. +It is used by a wide variety of Go projects, including [hugo][hugo] and [kops][kops]. ## "Normal" Usage From 2c9f4bef57b6278a853f00bb7a282bb7eb7e0b85 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:23:34 -0400 Subject: [PATCH 089/100] net.JoinHostPort Signed-off-by: Andrew Mason --- go/trace/plugin_datadog.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/trace/plugin_datadog.go b/go/trace/plugin_datadog.go index 2755742f9e0..b101607592d 100644 --- a/go/trace/plugin_datadog.go +++ b/go/trace/plugin_datadog.go @@ -3,6 +3,7 @@ package trace import ( "fmt" "io" + "net" "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" @@ -47,7 +48,7 @@ func newDatadogTracer(serviceName string) (tracingService, io.Closer, error) { } opts := []ddtracer.StartOption{ - ddtracer.WithAgentAddr(host + ":" + port), + ddtracer.WithAgentAddr(net.JoinHostPort(host, port)), ddtracer.WithServiceName(serviceName), ddtracer.WithDebugMode(true), ddtracer.WithSampler(ddtracer.NewRateSampler(samplingRate.Get())), From 88def226faab1edc91b08969690bb943419120cd Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:26:42 -0400 Subject: [PATCH 090/100] reorder imports Signed-off-by: Andrew Mason --- go/flags/endtoend/flags_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go/flags/endtoend/flags_test.go b/go/flags/endtoend/flags_test.go index 8199c52d1b4..61bc1dacfc3 100644 --- a/go/flags/endtoend/flags_test.go +++ b/go/flags/endtoend/flags_test.go @@ -22,15 +22,16 @@ package flags import ( "bytes" - _ "embed" "os" "os/exec" "testing" "text/template" - "vitess.io/vitess/go/test/utils" + _ "embed" "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/utils" ) var ( From 21e0350e9d5b569f1dc44fb4cf80147d4c7664bf Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:28:14 -0400 Subject: [PATCH 091/100] cleanup TODO panic Signed-off-by: Andrew Mason --- go/flagutil/enum.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/flagutil/enum.go b/go/flagutil/enum.go index 42f3b6d4f5a..6a733ee2d7f 100644 --- a/go/flagutil/enum.go +++ b/go/flagutil/enum.go @@ -81,7 +81,7 @@ func newStringEnum(name string, initialValue string, choices []string, caseInsen // This will panic if we've misconfigured something in the source code. // It's not a user-error, so it had damn-well be better caught by a test // somewhere. - panic("TODO: error message goes here") + panic(fmt.Errorf("%w (valid choices: %v)", ErrInvalidChoice, choiceNames)) } } From cb80b7b3a54743ae7863d1c74abd5d2116a35508 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:32:36 -0400 Subject: [PATCH 092/100] error string formatting Signed-off-by: Andrew Mason --- go/viperutil/config.go | 2 +- go/viperutil/debug/handler.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/viperutil/config.go b/go/viperutil/config.go index a551bbcf07c..d7d4cb6f8b9 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -86,7 +86,7 @@ var ( func init() { wd, err := os.Getwd() if err != nil { - log.WARN("failed to get working directory (err=%s), not appending to default config-paths", err) + log.WARN("failed to get working directory (err=%v), not appending to default config-paths", err) return } diff --git a/go/viperutil/debug/handler.go b/go/viperutil/debug/handler.go index f8d1c851731..07442dd13ab 100644 --- a/go/viperutil/debug/handler.go +++ b/go/viperutil/debug/handler.go @@ -62,18 +62,18 @@ func HandlerFunc(w http.ResponseWriter, r *http.Request) { v.SetConfigType(format) tmp, err := os.CreateTemp("", "viper_debug") if err != nil { - http.Error(w, fmt.Sprintf("failed to render config to tempfile: %s", err), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("failed to render config to tempfile: %v", err), http.StatusInternalServerError) return } defer os.Remove(tmp.Name()) if err := v.WriteConfigAs(tmp.Name()); err != nil { - http.Error(w, fmt.Sprintf("failed to render config to tempfile: %s", err), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("failed to render config to tempfile: %v", err), http.StatusInternalServerError) return } if _, err := io.Copy(w, tmp); err != nil { - http.Error(w, fmt.Sprintf("failed to write rendered config: %s", err), http.StatusInternalServerError) + http.Error(w, fmt.Sprintf("failed to write rendered config: %v", err), http.StatusInternalServerError) return } default: From 0598cbd2cd1fea826b4e59d522f4c78a0b0742f7 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:41:40 -0400 Subject: [PATCH 093/100] update comments Signed-off-by: Andrew Mason --- go/viperutil/get_func.go | 2 +- go/viperutil/internal/sync/sync.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go/viperutil/get_func.go b/go/viperutil/get_func.go index 98835cab07f..98362c32052 100644 --- a/go/viperutil/get_func.go +++ b/go/viperutil/get_func.go @@ -25,7 +25,7 @@ import ( "github.com/spf13/viper" ) -// GetFuncForType returns the default getter function for a given type T to. A +// GetFuncForType returns the default getter function for a given type T. A // getter function is a function which takes a viper and returns a function that // takes a key and (finally!) returns a value of type T. // diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index 929e20377b1..c689fa3f670 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -76,6 +76,11 @@ func (v *Viper) Set(key string, value any) { // We must not update v.disk here; explicit calls to Set will supercede all // future config reloads. v.live.Set(key, value) + + // Do a non-blocking signal to persist here. Our channel has a buffer of 1, + // so if we've signalled for some other Set call that hasn't been persisted + // yet, this Set will get persisted along with that one and any other + // pending in-memory changes. select { case v.setCh <- struct{}{}: default: From be0c895982a7eb0a4567d1c9c1d9134679bd7a44 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:43:40 -0400 Subject: [PATCH 094/100] copyright year update Signed-off-by: Andrew Mason --- go/flagutil/enum.go | 2 +- go/viperutil/config.go | 2 +- go/viperutil/config_test.go | 2 +- go/viperutil/debug/debug.go | 2 +- go/viperutil/errors.go | 2 +- go/viperutil/funcs/get.go | 2 +- go/viperutil/funcs/get_test.go | 2 +- go/viperutil/get_func.go | 2 +- go/viperutil/get_func_test.go | 2 +- go/viperutil/internal/log/log.go | 2 +- go/viperutil/internal/registry/registry.go | 2 +- go/viperutil/internal/sync/sync.go | 2 +- go/viperutil/internal/sync/sync_test.go | 2 +- go/viperutil/internal/value/value.go | 2 +- go/viperutil/value.go | 2 +- go/viperutil/viper.go | 2 +- go/viperutil/vipertest/stub.go | 2 +- 17 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go/flagutil/enum.go b/go/flagutil/enum.go index 6a733ee2d7f..5bc279ee493 100644 --- a/go/flagutil/enum.go +++ b/go/flagutil/enum.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/config.go b/go/viperutil/config.go index d7d4cb6f8b9..2af1f0c5419 100644 --- a/go/viperutil/config.go +++ b/go/viperutil/config.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/config_test.go b/go/viperutil/config_test.go index 00d4bc64eda..8e00a4700ac 100644 --- a/go/viperutil/config_test.go +++ b/go/viperutil/config_test.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/debug/debug.go b/go/viperutil/debug/debug.go index aa6acbd46ba..66cbc7f2962 100644 --- a/go/viperutil/debug/debug.go +++ b/go/viperutil/debug/debug.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/errors.go b/go/viperutil/errors.go index bb2018b4e88..5d18774f998 100644 --- a/go/viperutil/errors.go +++ b/go/viperutil/errors.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/funcs/get.go b/go/viperutil/funcs/get.go index 99022de9504..e33ffe9065f 100644 --- a/go/viperutil/funcs/get.go +++ b/go/viperutil/funcs/get.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/funcs/get_test.go b/go/viperutil/funcs/get_test.go index f20cc051a22..2af83e99aba 100644 --- a/go/viperutil/funcs/get_test.go +++ b/go/viperutil/funcs/get_test.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/get_func.go b/go/viperutil/get_func.go index 98362c32052..82a34df3fac 100644 --- a/go/viperutil/get_func.go +++ b/go/viperutil/get_func.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/get_func_test.go b/go/viperutil/get_func_test.go index 9692ae7b127..346573704f1 100644 --- a/go/viperutil/get_func_test.go +++ b/go/viperutil/get_func_test.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/internal/log/log.go b/go/viperutil/internal/log/log.go index 28a03d17289..cded335cd47 100644 --- a/go/viperutil/internal/log/log.go +++ b/go/viperutil/internal/log/log.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/internal/registry/registry.go b/go/viperutil/internal/registry/registry.go index 1250497e76c..10786a5f7b3 100644 --- a/go/viperutil/internal/registry/registry.go +++ b/go/viperutil/internal/registry/registry.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index c689fa3f670..c211c5fa611 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/internal/sync/sync_test.go b/go/viperutil/internal/sync/sync_test.go index a08d2ae1967..00159973ad2 100644 --- a/go/viperutil/internal/sync/sync_test.go +++ b/go/viperutil/internal/sync/sync_test.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/internal/value/value.go b/go/viperutil/internal/value/value.go index fbc754cb993..9feb5caba02 100644 --- a/go/viperutil/internal/value/value.go +++ b/go/viperutil/internal/value/value.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/value.go b/go/viperutil/value.go index e6efdc74888..4433b53b05d 100644 --- a/go/viperutil/value.go +++ b/go/viperutil/value.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go index fa4b8b5f7a4..7d120059587 100644 --- a/go/viperutil/viper.go +++ b/go/viperutil/viper.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/go/viperutil/vipertest/stub.go b/go/viperutil/vipertest/stub.go index e4673779454..5c8d3e78b43 100644 --- a/go/viperutil/vipertest/stub.go +++ b/go/viperutil/vipertest/stub.go @@ -1,5 +1,5 @@ /* -Copyright 2022 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 554c38dd566f32db21654d881b970b7480c06e0a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:43:55 -0400 Subject: [PATCH 095/100] delete empty file Signed-off-by: Andrew Mason --- go/viperutil/funcs/decode.go | 65 ------------------------------------ 1 file changed, 65 deletions(-) delete mode 100644 go/viperutil/funcs/decode.go diff --git a/go/viperutil/funcs/decode.go b/go/viperutil/funcs/decode.go deleted file mode 100644 index 723df555bcf..00000000000 --- a/go/viperutil/funcs/decode.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package funcs - -// TODO: this creates an import cycle ... IMO tabletenv should not define Seconds, -// it should be in something more akin to flagutil with the other values like -// TabletTypeFlag and friends. - -// import ( -// "reflect" -// "time" -// -// "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" -// ) -// -// func DecodeSeconds(from, to reflect.Type, data any) (any, error) { -// if to != reflect.TypeOf(tabletenv.Seconds(0)) { -// return data, nil -// } -// -// var ( -// n float64 -// ok bool -// ) -// switch from.Kind() { -// case reflect.Float32: -// data := data.(float32) -// n, ok = float64(data), true -// case reflect.Float64: -// n, ok = data.(float64), true -// } -// -// if ok { -// return tabletenv.Seconds(n), nil -// } -// -// if from != reflect.TypeOf(time.Duration(0)) { -// return data, nil -// } -// -// s := tabletenv.Seconds(0) -// s.Set(data.(time.Duration)) -// return s, nil -// } - -/* USAGE (similar to how ConfigFileNotFoundHandling is decoded in viperutil/config.go) -// viper.DecoderConfigOption(viper.DecodeHook(mapstructure.DecodeHookFunc(decode.Seconds))), -// viper.DecoderConfigOption(func(dc *mapstructure.DecoderConfig) { -// dc -// }), -*/ From 63ba9121ea20ebb585b868af0f9c792216541706 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 06:45:38 -0400 Subject: [PATCH 096/100] extract function for initializing the viper config Signed-off-by: Andrew Mason --- go/vt/servenv/servenv.go | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index 491d004e76e..662e4da5207 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -346,14 +346,7 @@ func ParseFlags(cmd string) { log.Exitf("%s doesn't take any positional arguments, got '%s'", cmd, strings.Join(args, " ")) } - watchCancel, err := viperutil.LoadConfig() - if err != nil { - log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) - } - OnTerm(watchCancel) - debugConfigRegisterOnce.Do(func() { - HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) - }) + loadViper(cmd) logutil.PurgeLogs() } @@ -387,6 +380,14 @@ func ParseFlagsWithArgs(cmd string) []string { log.Exitf("%s expected at least one positional argument", cmd) } + loadViper(cmd) + + logutil.PurgeLogs() + + return args +} + +func loadViper(cmd string) { watchCancel, err := viperutil.LoadConfig() if err != nil { log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) @@ -395,10 +396,6 @@ func ParseFlagsWithArgs(cmd string) []string { debugConfigRegisterOnce.Do(func() { HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) }) - - logutil.PurgeLogs() - - return args } // Flag installations for packages that servenv imports. We need to register From 52f94b46cb1227dac8e86af3d9b23789edd96f0a Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 13:36:38 -0400 Subject: [PATCH 097/100] update example package Signed-off-by: Andrew Mason --- doc/viper/viper.md | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index 659434c40e2..c9c3ab1f1f7 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -161,7 +161,7 @@ Therefore, we introduce a separate function, namely `viperutil.BindFlags`, which For example: ```go -package azblobbackupstorage +package trace import ( "github.com/spf13/pflag" @@ -171,36 +171,32 @@ import ( ) var ( - configKey = viperutil.KeyPrefixFunc("backup.storage.azblob") + configKey = viperutil.KeyPrefixFunc("trace") - accountName = viperutil.Configure( - configKey("account.name"), + tracingServer = viperutil.Configure( + configKey("service"), viperutil.Options[string]{ - EnvVars: []string{"VT_AZBLOB_ACCOUNT_NAME"}, - FlagName: "azblob_backup_account_name", + Default: "noop", + FlagName: "tracer", }, ) - - accountKeyFile = viperutil.Configure( - configKey("account.key_file"), - viperutil.Options[string]{ - FlagName: "azblob_backup_account_key_file", + enableLogging = viperutil.Configure( + configKey("enable-logging"), + viperutil.Options[bool]{ + FlagName: "tracing-enable-logging", }, ) ) -func registerFlags(fs *pflag.FlagSet) { - fs.String("azblob_backup_account_name", accountName.Default(), "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") - fs.String("azblob_backup_account_key_file", accountKeyFile.Default(), "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") +func RegisterFlags(fs *pflag.FlagSet) { + fs.String("tracer", tracingServer.Default(), "tracing service to use") + fs.Bool("tracing-enable-logging", false, "whether to enable logging in the tracing service") - viperutil.BindFlags(fs, accountName, accountKeyFile) + viperutil.BindFlags(fs, tracingServer, enableLogging) } func init() { - servenv.OnParseFor("vtbackup", registerFlags) - servenv.OnParseFor("vtctl", registerFlags) - servenv.OnParseFor("vtctld", registerFlags) - servenv.OnParseFor("vttablet", registerFlags) + servenv.OnParse(RegisterFlags) } ``` From 098f0d25efb4658d1ecf83d1e10f4b81f31323d7 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 13:41:45 -0400 Subject: [PATCH 098/100] more docs Signed-off-by: Andrew Mason --- doc/viper/viper.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/viper/viper.md b/doc/viper/viper.md index c9c3ab1f1f7..f9050ef842c 100644 --- a/doc/viper/viper.md +++ b/doc/viper/viper.md @@ -235,6 +235,12 @@ They are: - `Warn` => log at the WARNING level, but return no error. - `Error` => log at the ERROR level and return the error back to the caller (usually `servenv`.) - `Exit` => log at the FATAL level, exiting immediately. +- `--config-persistence-min-interval` + - Default: `1s` + - EnvVar: `VT_CONFIG_PERSISTENCE_MIN_INTERVAL` + - FlagType: `time.Duration` + - Behavior: If viper is watching a config file, in order to synchronize between changes to the file, and changes made in-memory to dynamic values (for example, via vtgate's `/debug/env` endpoint), it will periodically write in-memory changes back to disk, waiting _at least_ this long between writes. + If the value is 0, each in-memory `Set` is immediately followed by a write to disk. For more information on how viper searches for config files, see the [documentation][viper_read_in_config_docs]. From dee064f8ea55fb2e19a6e69500df32570e2b8418 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Tue, 23 May 2023 13:47:56 -0400 Subject: [PATCH 099/100] vterrors Signed-off-by: Andrew Mason --- go/viperutil/internal/sync/sync.go | 7 ++++--- go/viperutil/internal/value/value.go | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go index c211c5fa611..11cb028c286 100644 --- a/go/viperutil/internal/sync/sync.go +++ b/go/viperutil/internal/sync/sync.go @@ -18,7 +18,6 @@ package sync import ( "context" - "errors" "fmt" "sync" "time" @@ -28,6 +27,8 @@ import ( "github.com/spf13/viper" "vitess.io/vitess/go/viperutil/internal/log" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" ) // Viper is a wrapper around a pair of viper.Viper instances to provide config- @@ -89,7 +90,7 @@ func (v *Viper) Set(key string, value any) { // ErrDuplicateWatch is returned when Watch is called on a synced Viper which // has already started a watch. -var ErrDuplicateWatch = errors.New("duplicate watch") +var ErrDuplicateWatch = vterrors.New(vtrpc.Code_FAILED_PRECONDITION, "duplicate watch") // Watch starts watching the config used by the passed-in Viper. Before starting // the watch, the synced viper will perform an initial read and load from disk @@ -121,7 +122,7 @@ var ErrDuplicateWatch = errors.New("duplicate watch") // to ensure the config file can be read in properly. func (v *Viper) Watch(ctx context.Context, static *viper.Viper, minWaitInterval time.Duration) (cancel context.CancelFunc, err error) { if v.watchingConfig { - return nil, fmt.Errorf("%w: viper is already watching %s", ErrDuplicateWatch, v.disk.ConfigFileUsed()) + return nil, vterrors.Wrapf(ErrDuplicateWatch, "%s: viper is already watching %s", ErrDuplicateWatch.Error(), v.disk.ConfigFileUsed()) } ctx, cancel = context.WithCancel(ctx) diff --git a/go/viperutil/internal/value/value.go b/go/viperutil/internal/value/value.go index 9feb5caba02..a958d642bcf 100644 --- a/go/viperutil/internal/value/value.go +++ b/go/viperutil/internal/value/value.go @@ -17,7 +17,6 @@ limitations under the License. package value import ( - "errors" "fmt" "github.com/spf13/pflag" @@ -25,6 +24,8 @@ import ( "vitess.io/vitess/go/viperutil/internal/registry" "vitess.io/vitess/go/viperutil/internal/sync" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" ) // Registerable is the subset of the interface exposed by Values (which is @@ -59,7 +60,7 @@ func (val *Base[T]) Get() T { return val.BoundGetFunc(val.Key()) } // ErrNoFlagDefined is returned when a Value has a FlagName set, but the given // FlagSet does not define a flag with that name. -var ErrNoFlagDefined = errors.New("flag not defined") +var ErrNoFlagDefined = vterrors.New(vtrpc.Code_INVALID_ARGUMENT, "flag not defined") // Flag is part of the Registerable interface. If the given flag set has a flag // with the name of this value's configured flag, that flag is returned, along @@ -75,7 +76,7 @@ func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { flag := fs.Lookup(val.FlagName) if flag == nil { - return nil, fmt.Errorf("%w with name %s (for key %s)", ErrNoFlagDefined, val.FlagName, val.Key()) + return nil, vterrors.Wrapf(ErrNoFlagDefined, "%s with name %s (for key %s)", ErrNoFlagDefined.Error(), val.FlagName, val.Key()) } return flag, nil From 0da240ee0658566169113f9989590cca18e92316 Mon Sep 17 00:00:00 2001 From: Andrew Mason Date: Wed, 24 May 2023 06:36:11 -0400 Subject: [PATCH 100/100] bigger time interval Signed-off-by: Andrew Mason --- go/viperutil/internal/sync/sync_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/viperutil/internal/sync/sync_test.go b/go/viperutil/internal/sync/sync_test.go index 00159973ad2..fcafb941f48 100644 --- a/go/viperutil/internal/sync/sync_test.go +++ b/go/viperutil/internal/sync/sync_test.go @@ -87,7 +87,7 @@ func TestPersistConfig(t *testing.T) { t.Run("basic", func(t *testing.T) { v := vipersync.New() - minPersistWaitInterval := 2 * time.Second + minPersistWaitInterval := 10 * time.Second get := vipersync.AdaptGetter("foo", viperutil.GetFuncForType[int](), v) f, ch := setup(t, v, minPersistWaitInterval)