diff --git a/doc/viper/viper.md b/doc/viper/viper.md new file mode 100644 index 00000000000..f9050ef842c --- /dev/null +++ b/doc/viper/viper.md @@ -0,0 +1,324 @@ +# Vitess Viper Guidelines + +## What is Viper? + +[`viper`][viper] is a configuration-management library for Go programs. +It acts as a registry for configuration values coming from a variety of sources, including: + +- Default values. +- Configuration files (JSON, YAML, TOML, and other formats supported), including optionally watching and live-reloading. +- Environment variables. +- Command-line flags, primarily from `pflag.Flag` types. + +It is used by a wide variety of Go projects, including [hugo][hugo] and [kops][kops]. + +## "Normal" Usage + +Normally, and if you were to follow the examples on the viper documentation, you "just" load in a config file, maybe bind some flags, and then load values all across your codebase, like so: + +```go +// cmd/main.go +package main + +import ( + "log" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "example.com/pkg/stuff" +) + +func main() { + pflag.String("name", "", "name to print") + pflag.Parse() + + viper.AddConfigPath(".") + viper.AddConfigPath("/var/mypkg") + + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + log.Fatal(err) + } + } + + viper.BindPFlags(pflag.CommandLine) + viper.BindEnv("name", "MY_COOL_ENVVAR") + + stuff.Do() +} + +// pkg/stuff/do_stuff.go +package stuff + +import ( + "fmt" + + "github.com/spf13/viper" +) + +func Do() { + fmt.Println(viper.GetString("name")) +} +``` + +While this example is great for getting started with `viper` quickly — it is very easy, and very fast, to _write_ go from nothing to working code — it's not likely to scale well for a codebase the size and complexity of Vitess, for several reasons: + +1. Everything is globally-accessible. + + Currently, most of the config values in Vitess modules are un-exported (and we un-exported more of these during the `pflag` migration). + This is a good thing, as it gives each module control over how its configuration values are used, rather than allowing the raw values to leak across package boundaries. + +1. [Magical][a_theory_of_modern_go] access and lack of compile-time safety. + + In the above example, `package stuff` just "happens to know" that (1) `package main` binds a value to `viper` with the key `"name"` and (2) `"name"` is going to be bound to a `string` value specifically. + + If `package main` ever changes either of these two facts, `package stuff` is going to break (precisely how it breaks depends on what the changes are), and there's no way to catch this _before_ runtime without writing additional linters. (Note this is strongly-related to point 1 above). + +1. Hard to document. + + `viper` does not provide any sort of automatic documentation-generation code (we'll discuss this more later), so we will need to write our own tooling if we want to update our flag documentation with information like "this flag is also settable via this config key and these environment variables". + + If anyone anywhere can just magically try to read a value from the global registry without first declaring (1) that a config value with that key _should_ exist; (2) what flags, aliases, environment variables it reads from; and (3) what type it is, then writing that tooling is going to be vastly more complicated, and possibly impossible to do correctly. + +So, we use an approach that requires a bit more verbosity up-front to mitigate these drawbacks to the simpler approach. + +## Our Approach + +Instead of relying on the global `viper.Viper` singleton, we use a shim layer introduced in `package viperutil` to configure values in a standardized way across the entire Vitess codebase. + +This function, `Configure`, then returns a value object, with a `Get` method that returns the actual value from the viper registry. +Packages may then choose to export their config values, or not, as they see fit for their API consumers. + +### `Configure` Options + +In order to properly configure a value for use, `Configure` needs to know, broadly speaking, three things: + +1. The key name being bound. +1. What "things" it should be bound to (i.e. other keys via aliases, environment variables, and flag names), as well as if it has a default value. +1. How to `Get` it out of a viper. + +`Configure`, therefore, has the following signature: + +```go +func Configure[T any](key string, options Options[T]) Value[T] +``` + +The first parameter provides the key name (point 1 of our above list); all other information is provided via various `Options` fields, which looks like: + +```go +type Options[T any] struct { + // what "things" to bind to + Aliases []string + FlagName string + EnvVars []string + + // default, if any + Default T + + // whether it can reload or not (more on this later) + Dynamic bool + + // how to "get" it from a viper (more on this slightly less later) + GetFunc func(v *viper.Viper) func(key string) T +} +``` + +### `Get` funcs + +In most cases, module authors will not need to specify a `GetFunc` option, since, if not provided, `viperutil` will do its best to provide a sensible default for the given type `T`. + +This requires a fair amount of `reflect`ion code, which we won't go into here, and unfortunately cannot support even all primitive types (notably, array (not slice!!) types). +In these cases, the `GetFuncForType` will panic, allowing the module author to catch this during testing of their package. +They may then provide their own `GetFunc`. + +Authors may also want to provide their own `GetFunc` to provide additional logic to load a value even for types supported by `GetFuncForType` (for example, post-processing a string to ensure it is always lowercase). + +The full suite of types, both supported and panic-inducing, are documented by way of unit tests in [`go/viperutil/get_func_test.go`](../../go/viperutil/get_func_test.go). + +### Dynamic values + +Values can be configured to be either static or dynamic. +Static values are loaded once at startup (more precisely, when `viperutil.LoadConfig` is called), and whatever value is loaded at the point will be the result of calling `Get` on that value for the remainder of the process's lifetime. +Dynamic values, conversely, may respond to config changes. + +In order for dynamic configs to be truly dynamic, `LoadConfig` must have found a config file (as opposed to pulling values entirely from defaults, flags, and environment variables). +If this is the case, a second viper shim, which backs the dynamic registry, will start a watch on that file, and any changes to that file will be reflected in the `Get` methods of any values configured with `Dynamic: true`. + +**An important caveat** is that viper on its own is not threadsafe, meaning that if a config reload is being processed while a value is being accessed, a race condition can occur. +To protect against this, the dynamic registry uses a second shim, [`sync.Viper`](../../go/viperutil/internal/sync/sync.go). +This works by assigning each dynamic value its own `sync.RWMutex`, and locking it for writes whenever a config change is detected. Value `GetFunc`s are then adapted to wrap the underlying get in a `m.RLock(); defer m.RUnlock()` layer. +This means that there's a potential throughput impact of using dynamic values, which module authors should be aware of when deciding to make a given value dynamic. + +### A brief aside on flags + +In the name of "we will catch as many mistakes as possible in tests" ("mistakes" here referring to typos in flag names, deleting a flag in one place but forgetting to clean up another reference, and so on), `Values` will panic at bind-time if they are configured to bind to a flag name that does not exist. +Then, **as long as every binary is at least invoked** (including just `mycmd --help`) in an end-to-end test, our CI will fail if we ever misconfigure a value in this way. + +However, since `Configure` handles the binding of defaults, aliases, and envirnomnent variables, and is usually called in `var` blocks, this binding can actually happen before the module registers its flags via the `servenv.{OnParse,OnParseFor}` hooks. +If we were to also bind any named flags at the point of `Configure`, this would cause panics even if the module later registered a flag with that name. +Therefore, we introduce a separate function, namely `viperutil.BindFlags`, which binds the flags on one or more values, which modules can call _after_ registering their flags, usually in the same `OnParse` hook function. +For example: + +```go +package trace + +import ( + "github.com/spf13/pflag" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/vt/servenv" +) + +var ( + configKey = viperutil.KeyPrefixFunc("trace") + + tracingServer = viperutil.Configure( + configKey("service"), + viperutil.Options[string]{ + Default: "noop", + FlagName: "tracer", + }, + ) + enableLogging = viperutil.Configure( + configKey("enable-logging"), + viperutil.Options[bool]{ + FlagName: "tracing-enable-logging", + }, + ) +) + +func RegisterFlags(fs *pflag.FlagSet) { + fs.String("tracer", tracingServer.Default(), "tracing service to use") + fs.Bool("tracing-enable-logging", false, "whether to enable logging in the tracing service") + + viperutil.BindFlags(fs, tracingServer, enableLogging) +} + +func init() { + servenv.OnParse(RegisterFlags) +} +``` + +## Config Files + +`viperutil` provides a few flags that allow binaries to read values from config files in addition to defaults, environment variables and flags. +They are: + +- `--config-path` + - Default: `$(pwd)` + - EnvVar: `VT_CONFIG_PATH` (parsed exactly like a `$PATH` style shell variable). + - FlagType: `StringSlice` + - Behavior: Paths for `ReadInConfig` to search. +- `--config-type` + - Default: `""` + - EnvVar: `VT_CONFIG_TYPE` + - FlagType: `flagutil.StringEnum` + - Values: everything contained in `viper.SupportedExts`, case-insensitive. + - Behavior: Force viper to use a particular unmarshalling strategy; required if the config file does not have an extension (by default, viper infers the config type from the file extension). +- `--config-name` + - Default: `"vtconfig"` + - EnvVar: `VT_CONFIG_NAME` + - FlagType: `string` + - Behavior: Instructs `ReadInConfig` to only look in `ConfigPaths` for files named with this name (with any supported extension, unless `ConfigType` is also set, in which case only with that extension). +- `--config-file` + - Default: `""` + - EnvVar: `VT_CONFIG_FILE` + - FlagType: `string` + - Behavior: Instructs `ReadInConfig` to search in `ConfigPaths` for explicitly a file with this name. Takes precedence over `ConfigName`. +- `--config-file-not-found-handling` + - Default: `WarnOnConfigFileNotFound` + - EnvVar: (none) + - FlagType: `string` (options: `IgnoreConfigFileNotFound`, `WarnOnConfigFileNotFound`, `ErrorOnConfigFileNotFound`, `ExitOnConfigFileNotFound`) + - Behavior: If viper is unable to locate a config file (based on the other flags here), then `LoadConfig` will: + - `Ignore` => do nothing, return no error. Program values will come entirely from defaults, environment variables and flags. + - `Warn` => log at the WARNING level, but return no error. + - `Error` => log at the ERROR level and return the error back to the caller (usually `servenv`.) + - `Exit` => log at the FATAL level, exiting immediately. +- `--config-persistence-min-interval` + - Default: `1s` + - EnvVar: `VT_CONFIG_PERSISTENCE_MIN_INTERVAL` + - FlagType: `time.Duration` + - Behavior: If viper is watching a config file, in order to synchronize between changes to the file, and changes made in-memory to dynamic values (for example, via vtgate's `/debug/env` endpoint), it will periodically write in-memory changes back to disk, waiting _at least_ this long between writes. + If the value is 0, each in-memory `Set` is immediately followed by a write to disk. + +For more information on how viper searches for config files, see the [documentation][viper_read_in_config_docs]. + +If viper was able to locate and load a config file, `LoadConfig` will then configure the dynamic registry to set up a watch on that file, enabling all dynamic values to pick up changes to that file for the remainder of the program's execution. +If no config file was used, then dynamic values behave exactly like static values (i.e. the dynamic registry copies in the settings loaded into the static registry, but does not set up a file watch). + +### Re-persistence for Dynamic Values + +Prior to the introduction of viper in Vitess, certain components (such as `vttablet` or `vtgate`) exposed `/debug/env` HTTP endpoints that permitted the user to modify certain configuration parameters at runtime. + +This behavior is still supported, and to maintain consistency between update mechanisms, if: +- A config file was loaded at startup +- A value is configured with the `Dynamic: true` option + +then in-memory updates to that value (via `.Set()`) will be written back to disk. +If we skipped this step, then the next time viper reloaded the disk config, the in-memory change would be undone, since viper does a full load rather than something more differential. +Unfortunately, this seems unavoidable. + +To migitate against potentially writing to disk "too often" for a given user, the `--config-persistence-min-interval` flag defines the _minimum_ time to wait between writes. +Internally, the system is notified to write "soon" only when a dynamic value is updated. +If the wait period has elapsed between changes, a write happens immediately; otherwise, the system waits out the remainder of the period and persists any changes that happened while it was waiting. +Setting this interval to zero means that writes happen immediately. + +## Auto-Documentation + +One of the benefits of all values being created through a single function is that we can pretty easily build tooling to generate documentation for the config values available to a given binary. +The exact formatting can be tweaked, obviously, but as an example, something like: + +``` +{{ .BinaryName }} + +{{ range .Values }} +{{ .Key }}: + - Aliases: {{ join .Aliases ", " }} + - Environment Variables: {{ join .EnvVars ", " }} + {{- if hasFlag $.BinaryName .FlagName }} + - Flag: {{ .FlagName }} + {{ end -}} + {{- if hasDefault . }} + - Default: {{ .Default }} + {{ end -}} +{{ end }} +``` + +If/when we migrate other binaries to cobra, we can figure out how to combine this documntation with cobra's doc-generation tooling (which we use for `vtctldclient` and `vtadmin`). + +## Debug Endpoint + +Any component that parses its flags via one of `servenv`'s parsing methods will get an HTTP endpoint registered at `/debug/config` which displays the full viper configuration for debugging purposes. +It accepts a query parameter to control the format; anything in `viper.SupportedExts` is permitted. + +Components that do not use `servenv` to parse their flags may manually register the `(go/viperutil/debug).HandlerFunc` if they wish. + +## Caveats and Gotchas + +- Config keys are case-insensitive. +`Foo`, `foo`, `fOo`, and `FOO` will all have the same value. + - **Except** for environment variables, which, when read, are case-sensitive (but the config key they are _bound to_ remains case-insensitive). + For example, if you have `viper.BindEnv("foo", "VT_FOO")`, then `VT_FOO=1 ./myprogram` will set the value to `1`, but `Vt_FoO=1 ./myprogram will not`. + The value, though, can still be read _from_ viper as `Foo`, `foo`, `FOO`, and so on. + +- `Sub` is a split-brain. + The viper docs discuss using the `Sub` method on a viper to extract a subtree of a config to pass to a submodule. + This seems like a good idea, but has some fun surprises. + Each viper maintains its own settings map, and extracting a sub-tree creates a second settings map that is now completely divorced from the parent. + If you were to `parent.Set(key, value)`, the sub-viper will still have the old value. + Furthermore, if the parent was watching a config file for changes, the sub-viper is _not_ watching that file. + + For these reasons, we **strongly** discourage use of `v.Sub`. + +- The `Unmarshal*` functions rely on `mapstructure` tags, not `json|yaml|...` tags. + +- Any config files/paths added _after_ calling `WatchConfig` will not get picked up by that viper, and a viper can only watch a single config file. + +[viper]: https://github.com/spf13/viper +[viper_read_in_config_docs]: https://github.com/spf13/viper#reading-config-files + +[hugo]: https://github.com/gohugoio/hugo +[kops]: https://github.com/kubernetes/kops + +[a_theory_of_modern_go]: https://peter.bourgon.org/blog/2017/06/09/theory-of-modern-go.html diff --git a/go.mod b/go.mod index 140860a499e..dcfe435430f 100644 --- a/go.mod +++ b/go.mod @@ -107,7 +107,9 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/kr/pretty v0.3.1 github.com/kr/text v0.2.0 + github.com/mitchellh/mapstructure v1.5.0 github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 + github.com/spf13/jwalterweatherman v1.1.0 github.com/xlab/treeprint v1.2.0 golang.org/x/exp v0.0.0-20230131160201-f062dba9d201 golang.org/x/sync v0.1.0 @@ -164,7 +166,6 @@ require ( github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -179,7 +180,6 @@ require ( github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect diff --git a/go/flags/endtoend/flags_test.go b/go/flags/endtoend/flags_test.go index 1ad69c7fa84..61bc1dacfc3 100644 --- a/go/flags/endtoend/flags_test.go +++ b/go/flags/endtoend/flags_test.go @@ -22,13 +22,16 @@ package flags import ( "bytes" - _ "embed" + "os" "os/exec" "testing" + "text/template" - "vitess.io/vitess/go/test/utils" + _ "embed" "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/test/utils" ) var ( @@ -105,16 +108,30 @@ var ( ) func TestHelpOutput(t *testing.T) { + wd, err := os.Getwd() + require.NoError(t, err) + args := []string{"--help"} for binary, helptext := range helpOutput { t.Run(binary, func(t *testing.T) { + tmpl, err := template.New(binary).Parse(helptext) + require.NoError(t, err) + + var buf bytes.Buffer + err = tmpl.Execute(&buf, struct { + Workdir string + }{ + Workdir: wd, + }) + require.NoError(t, err) + cmd := exec.Command(binary, args...) output := bytes.Buffer{} cmd.Stderr = &output cmd.Stdout = &output - err := cmd.Run() + err = cmd.Run() require.NoError(t, err) - utils.MustMatch(t, helptext, output.String()) + utils.MustMatch(t, buf.String(), output.String()) }) } } diff --git a/go/flags/endtoend/mysqlctl.txt b/go/flags/endtoend/mysqlctl.txt index cdd64593649..2472528260b 100644 --- a/go/flags/endtoend/mysqlctl.txt +++ b/go/flags/endtoend/mysqlctl.txt @@ -11,71 +11,77 @@ The commands are listed below. Use 'mysqlctl -- {-h, --help}' for comm position Global flags: - --alsologtostderr log to standard error as well as files - --app_idle_timeout duration Idle timeout for app connections (default 1m0s) - --app_pool_size int Size of the connection pool for app connections (default 40) - --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified - --db-credentials-file string db credentials file; send SIGHUP to reload this file - --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") - --db-credentials-vault-addr string URL to Vault server - --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds - --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") - --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable - --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable - --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) - --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate - --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable - --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) - --db_charset string Character set used for this tablet. (default "utf8mb4") - --db_conn_query_info enable parsing and processing of QUERY_OK info fields - --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) - --db_dba_password string db dba password - --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) - --db_dba_user string db dba user userKey (default "vt_dba") - --db_flags uint Flag values as defined by MySQL. - --db_flavor string Flavor overrid. Valid value is FilePos. - --db_host string The host name for the tcp connection. - --db_port int tcp port - --db_server_name string server name of the DB we are connecting to. - --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. - --db_ssl_ca string connection ssl ca - --db_ssl_ca_path string connection ssl ca path - --db_ssl_cert string connection ssl certificate - --db_ssl_key string connection ssl key - --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. - --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. - --dba_idle_timeout duration Idle timeout for dba connections (default 1m0s) - --dba_pool_size int Size of the connection pool for dba connections (default 20) - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --max-stack-size int configure the maximum stack size in bytes (default 67108864) - --mysql_port int MySQL port (default 3306) - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_socket string Path to the mysqld socket file - --mysqlctl_client_protocol string the protocol to use to talk to the mysqlctl server (default "grpc") - --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init - --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) - --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) - --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) - --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. - --pool_hostname_resolve_interval duration if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled) - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --replication_connect_retry duration how long to wait in between replica reconnect attempts. Only precise to the second. (default 10s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice - --socket_file string Local unix socket file to listen on - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class - --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. - --tablet_uid uint32 Tablet UID (default 41983) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --alsologtostderr log to standard error as well as files + --app_idle_timeout duration Idle timeout for app connections (default 1m0s) + --app_pool_size int Size of the connection pool for app connections (default 40) + --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --db-credentials-file string db credentials file; send SIGHUP to reload this file + --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") + --db-credentials-vault-addr string URL to Vault server + --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds + --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") + --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable + --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable + --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) + --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate + --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable + --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) + --db_charset string Character set used for this tablet. (default "utf8mb4") + --db_conn_query_info enable parsing and processing of QUERY_OK info fields + --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) + --db_dba_password string db dba password + --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) + --db_dba_user string db dba user userKey (default "vt_dba") + --db_flags uint Flag values as defined by MySQL. + --db_flavor string Flavor overrid. Valid value is FilePos. + --db_host string The host name for the tcp connection. + --db_port int tcp port + --db_server_name string server name of the DB we are connecting to. + --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. + --db_ssl_ca string connection ssl ca + --db_ssl_ca_path string connection ssl ca path + --db_ssl_cert string connection ssl certificate + --db_ssl_key string connection ssl key + --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. + --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. + --dba_idle_timeout duration Idle timeout for dba connections (default 1m0s) + --dba_pool_size int Size of the connection pool for dba connections (default 20) + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --mysql_port int MySQL port (default 3306) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_socket string Path to the mysqld socket file + --mysqlctl_client_protocol string the protocol to use to talk to the mysqlctl server (default "grpc") + --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init + --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) + --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) + --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) + --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. + --pool_hostname_resolve_interval duration if set force an update to all hostnames and reconnect if changed, defaults to 0 (disabled) + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --replication_connect_retry duration how long to wait in between replica reconnect attempts. Only precise to the second. (default 10s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --service_map strings comma separated list of services to enable (or disable if prefixed with '-') Example: grpc-queryservice + --socket_file string Local unix socket file to listen on + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class + --tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid. + --tablet_uid uint32 Tablet UID (default 41983) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging diff --git a/go/flags/endtoend/mysqlctld.txt b/go/flags/endtoend/mysqlctld.txt index 09e3c144620..8b0ca101cb4 100644 --- a/go/flags/endtoend/mysqlctld.txt +++ b/go/flags/endtoend/mysqlctld.txt @@ -3,6 +3,12 @@ Usage of mysqlctld: --app_idle_timeout duration Idle timeout for app connections (default 1m0s) --app_pool_size int Size of the connection pool for app connections (default 40) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --db-credentials-file string db credentials file; send SIGHUP to reload this file --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") --db-credentials-vault-addr string URL to Vault server diff --git a/go/flags/endtoend/vtaclcheck.txt b/go/flags/endtoend/vtaclcheck.txt index 6e2c57db109..d28eb7351b9 100644 --- a/go/flags/endtoend/vtaclcheck.txt +++ b/go/flags/endtoend/vtaclcheck.txt @@ -1,19 +1,25 @@ Usage of vtaclcheck: - --acl-file string The path of the JSON ACL file to check - --alsologtostderr log to standard error as well as files - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --static-auth-file string The path of the auth_server_static JSON file to check - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --acl-file string The path of the JSON ACL file to check + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --static-auth-file string The path of the auth_server_static JSON file to check + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index ded70c2550a..f825e764e6c 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -1,188 +1,194 @@ Usage of vtbackup: - --allow_first_backup Allow this job to take the first backup of an existing shard. - --alsologtostderr log to standard error as well as files - --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). - --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. - --azblob_backup_container_name string Azure Blob Container Name. - --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) - --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). - --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") - --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) - --backup_storage_compress if set, the backup files will be compressed. (default true) - --backup_storage_implementation string Which backup storage implementation to use for creating and restoring backups. - --backup_storage_number_blocks int if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2) - --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. - --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) - --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) - --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) - --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") - --compression-engine-name string compressor engine used for compression. (default "pargzip") - --compression-level int what level to pass to the compressor. (default 1) - --concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) - --consul_auth_static_file string JSON File to read the topos/tokens from. - --db-credentials-file string db credentials file; send SIGHUP to reload this file - --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") - --db-credentials-vault-addr string URL to Vault server - --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds - --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") - --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable - --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable - --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) - --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate - --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable - --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) - --db_allprivs_password string db allprivs password - --db_allprivs_use_ssl Set this flag to false to make the allprivs connection to not use ssl (default true) - --db_allprivs_user string db allprivs user userKey (default "vt_allprivs") - --db_app_password string db app password - --db_app_use_ssl Set this flag to false to make the app connection to not use ssl (default true) - --db_app_user string db app user userKey (default "vt_app") - --db_appdebug_password string db appdebug password - --db_appdebug_use_ssl Set this flag to false to make the appdebug connection to not use ssl (default true) - --db_appdebug_user string db appdebug user userKey (default "vt_appdebug") - --db_charset string Character set used for this tablet. (default "utf8mb4") - --db_conn_query_info enable parsing and processing of QUERY_OK info fields - --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) - --db_dba_password string db dba password - --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) - --db_dba_user string db dba user userKey (default "vt_dba") - --db_erepl_password string db erepl password - --db_erepl_use_ssl Set this flag to false to make the erepl connection to not use ssl (default true) - --db_erepl_user string db erepl user userKey (default "vt_erepl") - --db_filtered_password string db filtered password - --db_filtered_use_ssl Set this flag to false to make the filtered connection to not use ssl (default true) - --db_filtered_user string db filtered user userKey (default "vt_filtered") - --db_flags uint Flag values as defined by MySQL. - --db_flavor string Flavor overrid. Valid value is FilePos. - --db_host string The host name for the tcp connection. - --db_port int tcp port - --db_repl_password string db repl password - --db_repl_use_ssl Set this flag to false to make the repl connection to not use ssl (default true) - --db_repl_user string db repl user userKey (default "vt_repl") - --db_server_name string server name of the DB we are connecting to. - --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. - --db_ssl_ca string connection ssl ca - --db_ssl_ca_path string connection ssl ca path - --db_ssl_cert string connection ssl certificate - --db_ssl_key string connection ssl key - --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. - --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. - --detach detached mode - run backups detached from the terminal - --disable-redo-log Disable InnoDB redo log during replication-from-primary phase of backup. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --external-compressor string command with arguments to use when compressing a backup. - --external-compressor-extension string extension to use when using an external compressor. - --external-decompressor string command with arguments to use when decompressing a backup. - --file_backup_storage_root string Root directory for the file backup storage. - --gcs_backup_storage_bucket string Google Cloud Storage bucket to use for backups. - --gcs_backup_storage_root string Root prefix for all backup-related object names. - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --incremental_from_pos string Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position - --init_db_name_override string (init parameter) override the name of the db used by vttablet - --init_db_sql_file string path to .sql file to run after mysql_install_db - --init_keyspace string (init parameter) keyspace to use for this tablet - --init_shard string (init parameter) shard to use for this tablet - --initial_backup Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed). - --keep-alive-timeout duration Wait until timeout elapses after a successful backup before shutting down. - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --manifest-external-decompressor string command with arguments to store in the backup manifest when compressing a backup with an external compression engine. - --min_backup_interval duration Only take a new backup if it's been at least this long since the most recent backup. - --min_retention_count int Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made (default 1) - --min_retention_time duration Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups. - --mycnf-file string path to my.cnf, if reading all config params from there - --mycnf_bin_log_path string mysql binlog path - --mycnf_data_dir string data directory for mysql - --mycnf_error_log_path string mysql error log path - --mycnf_general_log_path string mysql general log path - --mycnf_innodb_data_home_dir string Innodb data home directory - --mycnf_innodb_log_group_home_dir string Innodb log group home directory - --mycnf_master_info_file string mysql master.info file - --mycnf_mysql_port int port mysql is listening on - --mycnf_pid_file string mysql pid file - --mycnf_relay_log_index_path string mysql relay log index path - --mycnf_relay_log_info_path string mysql relay log info path - --mycnf_relay_log_path string mysql relay log path - --mycnf_secure_file_priv string mysql path for loading secure files - --mycnf_server_id int mysql server id of the server (if specified, mycnf-file will be ignored) - --mycnf_slow_log_path string mysql slow query log path - --mycnf_socket_file string mysql socket file - --mycnf_tmp_dir string mysql tmp directory - --mysql_port int mysql port (default 3306) - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_socket string path to the mysql socket - --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) - --port int port for the server - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --restart_before_backup Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs. - --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). - --s3_backup_aws_region string AWS region to use. (default "us-east-1") - --s3_backup_aws_retries int AWS request retries. (default -1) - --s3_backup_force_path_style force the s3 path style. - --s3_backup_log_level string determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors. (default "LogOff") - --s3_backup_server_side_encryption string server-side encryption algorithm (e.g., AES256, aws:kms, sse_c:/path/to/key/file). - --s3_backup_storage_bucket string S3 bucket to use for backups. - --s3_backup_storage_root string root prefix for all backup-related object names. - --s3_backup_tls_skip_verify_cert skip the 'certificate is valid' check for SSL connections. - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) - --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --xbstream_restore_flags string Flags to pass to xbstream command during restore. These should be space separated and will be added to the end of the command. These need to match the ones used for backup e.g. --compress / --decompress, --encrypt / --decrypt - --xtrabackup_backup_flags string Flags to pass to backup command. These should be space separated and will be added to the end of the command - --xtrabackup_prepare_flags string Flags to pass to prepare command. These should be space separated and will be added to the end of the command - --xtrabackup_root_path string Directory location of the xtrabackup and xbstream executables, e.g., /usr/bin - --xtrabackup_stream_mode string Which mode to use if streaming, valid values are tar and xbstream. Please note that tar is not supported in XtraBackup 8.0 (default "tar") - --xtrabackup_stripe_block_size uint Size in bytes of each block that gets sent to a given stripe before rotating to the next stripe (default 102400) - --xtrabackup_stripes uint If greater than 0, use data striping across this many destination files to parallelize data transfer and decompression - --xtrabackup_user string User that xtrabackup will use to connect to the database server. This user must have all necessary privileges. For details, please refer to xtrabackup documentation. + --allow_first_backup Allow this job to take the first backup of an existing shard. + --alsologtostderr log to standard error as well as files + --azblob_backup_account_key_file string Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path). + --azblob_backup_account_name string Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used. + --azblob_backup_container_name string Azure Blob Container Name. + --azblob_backup_parallelism int Azure Blob operation parallelism (requires extra memory when increased). (default 1) + --azblob_backup_storage_root string Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/'). + --backup_engine_implementation string Specifies which implementation to use for creating new backups (builtin or xtrabackup). Restores will always be done with whichever engine created a given backup. (default "builtin") + --backup_storage_block_size int if backup_storage_compress is true, backup_storage_block_size sets the byte size for each block while compressing (default is 250000). (default 250000) + --backup_storage_compress if set, the backup files will be compressed. (default true) + --backup_storage_implementation string Which backup storage implementation to use for creating and restoring backups. + --backup_storage_number_blocks int if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2) + --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. + --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) + --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) + --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") + --compression-engine-name string compressor engine used for compression. (default "pargzip") + --compression-level int what level to pass to the compressor. (default 1) + --concurrency int (init restore parameter) how many concurrent files to restore at once (default 4) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --consul_auth_static_file string JSON File to read the topos/tokens from. + --db-credentials-file string db credentials file; send SIGHUP to reload this file + --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") + --db-credentials-vault-addr string URL to Vault server + --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds + --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") + --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable + --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable + --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) + --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate + --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable + --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) + --db_allprivs_password string db allprivs password + --db_allprivs_use_ssl Set this flag to false to make the allprivs connection to not use ssl (default true) + --db_allprivs_user string db allprivs user userKey (default "vt_allprivs") + --db_app_password string db app password + --db_app_use_ssl Set this flag to false to make the app connection to not use ssl (default true) + --db_app_user string db app user userKey (default "vt_app") + --db_appdebug_password string db appdebug password + --db_appdebug_use_ssl Set this flag to false to make the appdebug connection to not use ssl (default true) + --db_appdebug_user string db appdebug user userKey (default "vt_appdebug") + --db_charset string Character set used for this tablet. (default "utf8mb4") + --db_conn_query_info enable parsing and processing of QUERY_OK info fields + --db_connect_timeout_ms int connection timeout to mysqld in milliseconds (0 for no timeout) + --db_dba_password string db dba password + --db_dba_use_ssl Set this flag to false to make the dba connection to not use ssl (default true) + --db_dba_user string db dba user userKey (default "vt_dba") + --db_erepl_password string db erepl password + --db_erepl_use_ssl Set this flag to false to make the erepl connection to not use ssl (default true) + --db_erepl_user string db erepl user userKey (default "vt_erepl") + --db_filtered_password string db filtered password + --db_filtered_use_ssl Set this flag to false to make the filtered connection to not use ssl (default true) + --db_filtered_user string db filtered user userKey (default "vt_filtered") + --db_flags uint Flag values as defined by MySQL. + --db_flavor string Flavor overrid. Valid value is FilePos. + --db_host string The host name for the tcp connection. + --db_port int tcp port + --db_repl_password string db repl password + --db_repl_use_ssl Set this flag to false to make the repl connection to not use ssl (default true) + --db_repl_user string db repl user userKey (default "vt_repl") + --db_server_name string server name of the DB we are connecting to. + --db_socket string The unix socket to connect on. If this is specified, host and port will not be used. + --db_ssl_ca string connection ssl ca + --db_ssl_ca_path string connection ssl ca path + --db_ssl_cert string connection ssl certificate + --db_ssl_key string connection ssl key + --db_ssl_mode SslMode SSL mode to connect with. One of disabled, preferred, required, verify_ca & verify_identity. + --db_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. + --detach detached mode - run backups detached from the terminal + --disable-redo-log Disable InnoDB redo log during replication-from-primary phase of backup. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --external-compressor string command with arguments to use when compressing a backup. + --external-compressor-extension string extension to use when using an external compressor. + --external-decompressor string command with arguments to use when decompressing a backup. + --file_backup_storage_root string Root directory for the file backup storage. + --gcs_backup_storage_bucket string Google Cloud Storage bucket to use for backups. + --gcs_backup_storage_root string Root prefix for all backup-related object names. + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --incremental_from_pos string Position of previous backup. Default: empty. If given, then this backup becomes an incremental backup from given position. If value is 'auto', backup taken from last successful backup position + --init_db_name_override string (init parameter) override the name of the db used by vttablet + --init_db_sql_file string path to .sql file to run after mysql_install_db + --init_keyspace string (init parameter) keyspace to use for this tablet + --init_shard string (init parameter) shard to use for this tablet + --initial_backup Instead of restoring from backup, initialize an empty database with the provided init_db_sql_file and upload a backup of that for the shard, if the shard has no backups yet. This can be used to seed a brand new shard with an initial, empty backup. If any backups already exist for the shard, this will be considered a successful no-op. This can only be done before the shard exists in topology (i.e. before any tablets are deployed). + --keep-alive-timeout duration Wait until timeout elapses after a successful backup before shutting down. + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --manifest-external-decompressor string command with arguments to store in the backup manifest when compressing a backup with an external compression engine. + --min_backup_interval duration Only take a new backup if it's been at least this long since the most recent backup. + --min_retention_count int Always keep at least this many of the most recent backups in this backup storage location, even if some are older than the min_retention_time. This must be at least 1 since a backup must always exist to allow new backups to be made (default 1) + --min_retention_time duration Keep each old backup for at least this long before removing it. Set to 0 to disable pruning of old backups. + --mycnf-file string path to my.cnf, if reading all config params from there + --mycnf_bin_log_path string mysql binlog path + --mycnf_data_dir string data directory for mysql + --mycnf_error_log_path string mysql error log path + --mycnf_general_log_path string mysql general log path + --mycnf_innodb_data_home_dir string Innodb data home directory + --mycnf_innodb_log_group_home_dir string Innodb log group home directory + --mycnf_master_info_file string mysql master.info file + --mycnf_mysql_port int port mysql is listening on + --mycnf_pid_file string mysql pid file + --mycnf_relay_log_index_path string mysql relay log index path + --mycnf_relay_log_info_path string mysql relay log info path + --mycnf_relay_log_path string mysql relay log path + --mycnf_secure_file_priv string mysql path for loading secure files + --mycnf_server_id int mysql server id of the server (if specified, mycnf-file will be ignored) + --mycnf_slow_log_path string mysql slow query log path + --mycnf_socket_file string mysql socket file + --mycnf_tmp_dir string mysql tmp directory + --mysql_port int mysql port (default 3306) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_socket string path to the mysql socket + --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) + --port int port for the server + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --restart_before_backup Perform a mysqld clean/full restart after applying binlogs, but before taking the backup. Only makes sense to work around xtrabackup bugs. + --s3_backup_aws_endpoint string endpoint of the S3 backend (region must be provided). + --s3_backup_aws_region string AWS region to use. (default "us-east-1") + --s3_backup_aws_retries int AWS request retries. (default -1) + --s3_backup_force_path_style force the s3 path style. + --s3_backup_log_level string determine the S3 loglevel to use from LogOff, LogDebug, LogDebugWithSigning, LogDebugWithHTTPBody, LogDebugWithRequestRetries, LogDebugWithRequestErrors. (default "LogOff") + --s3_backup_server_side_encryption string server-side encryption algorithm (e.g., AES256, aws:kms, sse_c:/path/to/key/file). + --s3_backup_storage_bucket string S3 bucket to use for backups. + --s3_backup_storage_root string root prefix for all backup-related object names. + --s3_backup_tls_skip_verify_cert skip the 'certificate is valid' check for SSL connections. + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) + --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --xbstream_restore_flags string Flags to pass to xbstream command during restore. These should be space separated and will be added to the end of the command. These need to match the ones used for backup e.g. --compress / --decompress, --encrypt / --decrypt + --xtrabackup_backup_flags string Flags to pass to backup command. These should be space separated and will be added to the end of the command + --xtrabackup_prepare_flags string Flags to pass to prepare command. These should be space separated and will be added to the end of the command + --xtrabackup_root_path string Directory location of the xtrabackup and xbstream executables, e.g., /usr/bin + --xtrabackup_stream_mode string Which mode to use if streaming, valid values are tar and xbstream. Please note that tar is not supported in XtraBackup 8.0 (default "tar") + --xtrabackup_stripe_block_size uint Size in bytes of each block that gets sent to a given stripe before rotating to the next stripe (default 102400) + --xtrabackup_stripes uint If greater than 0, use data striping across this many destination files to parallelize data transfer and decompression + --xtrabackup_user string User that xtrabackup will use to connect to the database server. This user must have all necessary privileges. For details, please refer to xtrabackup documentation. diff --git a/go/flags/endtoend/vtctlclient.txt b/go/flags/endtoend/vtctlclient.txt index 207f31905f2..cbb6d74cb9d 100644 --- a/go/flags/endtoend/vtctlclient.txt +++ b/go/flags/endtoend/vtctlclient.txt @@ -1,41 +1,47 @@ Usage of vtctlclient: - --action_timeout duration timeout for the total command (default 1h0m0s) - --alsologtostderr log to standard error as well as files - --datadog-agent-host string host to send spans to. if empty, no tracing will be done - --datadog-agent-port string port to send spans to. if empty, no tracing will be done - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --jaeger-agent-host string host and port to send spans to. if empty, no tracing will be done - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --server string server to use for connection - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --tracer string tracing service to use (default "noop") - --tracing-enable-logging whether to enable logging in the tracing service - --tracing-sampling-rate float sampling rate for the probabilistic jaeger sampler (default 0.1) - --tracing-sampling-type string sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote' (default "const") - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --vtctl_client_protocol string Protocol to use to talk to the vtctl server. (default "grpc") - --vtctld_grpc_ca string the server ca to use to validate servers when connecting - --vtctld_grpc_cert string the cert to use to connect - --vtctld_grpc_crl string the server crl to use to validate server certificates when connecting - --vtctld_grpc_key string the key to use to connect - --vtctld_grpc_server_name string the server name to use to validate server certificate + --action_timeout duration timeout for the total command (default 1h0m0s) + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --datadog-agent-host string host to send spans to. if empty, no tracing will be done + --datadog-agent-port string port to send spans to. if empty, no tracing will be done + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --jaeger-agent-host string host and port to send spans to. if empty, no tracing will be done + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --server string server to use for connection + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --tracer string tracing service to use (default "noop") + --tracing-enable-logging whether to enable logging in the tracing service + --tracing-sampling-rate float sampling rate for the probabilistic jaeger sampler (default 0.1) + --tracing-sampling-type string sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote' (default "const") + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vtctl_client_protocol string Protocol to use to talk to the vtctl server. (default "grpc") + --vtctld_grpc_ca string the server ca to use to validate servers when connecting + --vtctld_grpc_cert string the cert to use to connect + --vtctld_grpc_crl string the server crl to use to validate server certificates when connecting + --vtctld_grpc_key string the key to use to connect + --vtctld_grpc_server_name string the server name to use to validate server certificate diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index eb8c0c552dc..7bc57f53630 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -18,6 +18,12 @@ Usage of vtctld: --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --cell string cell to use --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. --datadog-agent-host string host to send spans to. if empty, no tracing will be done --datadog-agent-port string port to send spans to. if empty, no tracing will be done @@ -115,7 +121,7 @@ Usage of vtctld: --tablet_protocol string Protocol to use to make queryservice RPCs to vttablets. (default "grpc") --tablet_refresh_interval duration Tablet refresh interval. (default 1m0s) --tablet_refresh_known_tablets Whether to reload the tablet's address/port map from topo in case they change. (default true) - --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{.GetTabletHostPort}}") + --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}") --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. diff --git a/go/flags/endtoend/vtexplain.txt b/go/flags/endtoend/vtexplain.txt index a70067a9bc8..29ffb35f343 100644 --- a/go/flags/endtoend/vtexplain.txt +++ b/go/flags/endtoend/vtexplain.txt @@ -1,37 +1,43 @@ Usage of vtexplain: - --alsologtostderr log to standard error as well as files - --batch-interval duration Interval between logical time slots. (default 10ms) - --dbname string Optional database target to override normal routing - --default_tablet_type topodatapb.TabletType The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY) - --execution-mode string The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc (default "multi") - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --ks-shard-map string JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace - --ks-shard-map-file string File containing json blob of keyspace name -> shard name -> ShardReference object - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --normalize Whether to enable vtgate normalization - --output-mode string Output in human-friendly text or json (default "text") - --planner-version string Sets the query planner version to use when generating the explain output. Valid values are V3 and Gen4. An empty value will use VTGate's default planner - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --replication-mode string The replication mode to simulate -- must be set to either ROW or STATEMENT (default "ROW") - --schema string The SQL table schema - --schema-file string Identifies the file that contains the SQL table schema - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --shards int Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored. (default 2) - --sql string A list of semicolon-delimited SQL commands to analyze - --sql-file string Identifies the file that contains the SQL commands to analyze - --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) - --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --vschema string Identifies the VTGate routing schema - --vschema-file string Identifies the VTGate routing schema file + --alsologtostderr log to standard error as well as files + --batch-interval duration Interval between logical time slots. (default 10ms) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --dbname string Optional database target to override normal routing + --default_tablet_type topodatapb.TabletType The default tablet type to set for queries, when one is not explicitly selected. (default PRIMARY) + --execution-mode string The execution mode to simulate -- must be set to multi, legacy-autocommit, or twopc (default "multi") + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --ks-shard-map string JSON map of keyspace name -> shard name -> ShardReference object. The inner map is the same as the output of FindAllShardsInKeyspace + --ks-shard-map-file string File containing json blob of keyspace name -> shard name -> ShardReference object + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --normalize Whether to enable vtgate normalization + --output-mode string Output in human-friendly text or json (default "text") + --planner-version string Sets the query planner version to use when generating the explain output. Valid values are V3 and Gen4. An empty value will use VTGate's default planner + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --replication-mode string The replication mode to simulate -- must be set to either ROW or STATEMENT (default "ROW") + --schema string The SQL table schema + --schema-file string Identifies the file that contains the SQL table schema + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --shards int Number of shards per keyspace. Passing --ks-shard-map/--ks-shard-map-file causes this flag to be ignored. (default 2) + --sql string A list of semicolon-delimited SQL commands to analyze + --sql-file string Identifies the file that contains the SQL commands to analyze + --sql-max-length-errors int truncate queries in error logs to the given length (default unlimited) + --sql-max-length-ui int truncate queries in debug UIs to the given length (default 512) (default 512) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vschema string Identifies the VTGate routing schema + --vschema-file string Identifies the VTGate routing schema file diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index 842aa288779..bb576961c29 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -11,6 +11,12 @@ Usage of vtgate: --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified --cell string cell to use --cells_to_watch string comma-separated list of cells for watching tablets + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. --datadog-agent-host string host to send spans to. if empty, no tracing will be done --datadog-agent-port string port to send spans to. if empty, no tracing will be done @@ -168,7 +174,7 @@ Usage of vtgate: --tablet_refresh_interval duration Tablet refresh interval. (default 1m0s) --tablet_refresh_known_tablets Whether to reload the tablet's address/port map from topo in case they change. (default true) --tablet_types_to_wait strings Wait till connected for specified tablet types during Gateway initialization. Should be provided as a comma-separated set of tablet types. - --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{.GetTabletHostPort}}") + --tablet_url_template string Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}") --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") --topo_consul_lock_session_ttl string TTL for consul session. diff --git a/go/flags/endtoend/vtgr.txt b/go/flags/endtoend/vtgr.txt index b70b84d21f3..c553ee8015c 100644 --- a/go/flags/endtoend/vtgr.txt +++ b/go/flags/endtoend/vtgr.txt @@ -1,84 +1,90 @@ Usage of vtgr: - --abort_rebootstrap Don't allow vtgr to rebootstrap an existing group. - --alsologtostderr log to standard error as well as files - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" - --consul_auth_static_file string JSON File to read the topos/tokens from. - --db-credentials-file string db credentials file; send SIGHUP to reload this file - --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") - --db-credentials-vault-addr string URL to Vault server - --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds - --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") - --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable - --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable - --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) - --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate - --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable - --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) - --db_config string Full path to db config file that will be used by VTGR. - --db_flavor string MySQL flavor override. (default "MySQL56") - --db_port int Local mysql port, set this to enable local fast check. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --enable_heartbeat_check Enable heartbeat checking, set together with --group_heartbeat_threshold. - --gr_port int Port to bootstrap a MySQL group. (default 33061) - --group_heartbeat_threshold int VTGR will trigger backoff on inconsistent state if the group heartbeat staleness exceeds this threshold (in seconds). Should be used along with --enable_heartbeat_check. - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --ping_tablet_timeout duration time to wait when we ping a tablet (default 2s) - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --refresh_interval duration Refresh interval to load tablets. (default 10s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --scan_interval duration Scan interval to diagnose and repair. (default 3s) - --scan_repair_timeout duration Time to wait for a Diagnose and repair operation. (default 3s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --vtgr_config string Config file for vtgr. + --abort_rebootstrap Don't allow vtgr to rebootstrap an existing group. + --alsologtostderr log to standard error as well as files + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --consul_auth_static_file string JSON File to read the topos/tokens from. + --db-credentials-file string db credentials file; send SIGHUP to reload this file + --db-credentials-server string db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation) (default "file") + --db-credentials-vault-addr string URL to Vault server + --db-credentials-vault-path string Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds + --db-credentials-vault-role-mountpoint string Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable (default "approle") + --db-credentials-vault-role-secretidfile string Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable + --db-credentials-vault-roleid string Vault AppRole id; can also be passed using VAULT_ROLEID environment variable + --db-credentials-vault-timeout duration Timeout for vault API operations (default 10s) + --db-credentials-vault-tls-ca string Path to CA PEM for validating Vault server certificate + --db-credentials-vault-tokenfile string Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable + --db-credentials-vault-ttl duration How long to cache DB credentials from the Vault server (default 30m0s) + --db_config string Full path to db config file that will be used by VTGR. + --db_flavor string MySQL flavor override. (default "MySQL56") + --db_port int Local mysql port, set this to enable local fast check. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --enable_heartbeat_check Enable heartbeat checking, set together with --group_heartbeat_threshold. + --gr_port int Port to bootstrap a MySQL group. (default 33061) + --group_heartbeat_threshold int VTGR will trigger backoff on inconsistent state if the group heartbeat staleness exceeds this threshold (in seconds). Should be used along with --enable_heartbeat_check. + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --ping_tablet_timeout duration time to wait when we ping a tablet (default 2s) + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --refresh_interval duration Refresh interval to load tablets. (default 10s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --scan_interval duration Scan interval to diagnose and repair. (default 3s) + --scan_repair_timeout duration Time to wait for a Diagnose and repair operation. (default 3s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --vtgr_config string Config file for vtgr. diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt index 60af343eca9..b38cca7374b 100644 --- a/go/flags/endtoend/vtorc.txt +++ b/go/flags/endtoend/vtorc.txt @@ -1,87 +1,93 @@ Usage of vtorc: - --alsologtostderr log to standard error as well as files - --audit-file-location string File location where the audit logs are to be stored - --audit-purge-duration duration Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s) - --audit-to-backend Whether to store the audit log in the VTOrc database - --audit-to-syslog Whether to store the audit log in the syslog - --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified - --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" - --config string config file name - --consul_auth_static_file string JSON File to read the topos/tokens from. - --emit_stats If set, emit stats to push-based monitoring and stats backends - --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. - --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy - --grpc_enable_tracing Enable gRPC tracing. - --grpc_initial_conn_window_size int gRPC initial connection window size - --grpc_initial_window_size int gRPC initial window size - --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) - --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) - --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) - --grpc_prometheus Enable gRPC monitoring with Prometheus. - -h, --help display usage and exit - --instance-poll-time duration Timer duration on which VTOrc refreshes MySQL information (default 5s) - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) - --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --max-stack-size int configure the maximum stack size in bytes (default 67108864) - --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) - --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) - --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. - --port int port for the server - --pprof strings enable profiling - --prevent-cross-cell-failover Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --reasonable-replication-lag duration Maximum replication lag on replicas which is deemed to be acceptable (default 10s) - --recovery-period-block-duration duration Duration for which a new recovery is blocked on an instance after running a recovery (default 30s) - --recovery-poll-duration duration Timer duration on which VTOrc polls its database to run a recovery (default 1s) - --remote_operation_timeout duration time to wait for a remote operation (default 15s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --shutdown_wait_time duration Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s) - --snapshot-topology-interval duration Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours - --sqlite-data-file string SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared") - --stats_backend string The name of the registered push-based monitoring/stats backend to use - --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars - --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 - --stats_drop_variables string Variables to be dropped from the list of exported variables. - --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class - --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting - --tablet_manager_grpc_cert string the cert to use to connect - --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) - --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) - --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting - --tablet_manager_grpc_key string the key to use to connect - --tablet_manager_grpc_server_name string the server name to use to validate server certificate - --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") - --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) - --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) - --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") - --topo_consul_lock_session_ttl string TTL for consul session. - --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) - --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) - --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server - --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS - --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS - --topo_global_root string the path of the global topology data in the global topology server - --topo_global_server_address string the address of the global topology server - --topo_implementation string the topology implementation to use - --topo_k8s_context string The kubeconfig context to use, overrides the 'current-context' from the config - --topo_k8s_kubeconfig string Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod - --topo_k8s_namespace string The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config - --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass - --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) - --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) - --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server - --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS - --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) + --alsologtostderr log to standard error as well as files + --audit-file-location string File location where the audit logs are to be stored + --audit-purge-duration duration Duration for which audit logs are held before being purged. Should be in multiples of days (default 168h0m0s) + --audit-to-backend Whether to store the audit log in the VTOrc database + --audit-to-syslog Whether to store the audit log in the syslog + --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified + --clusters_to_watch strings Comma-separated list of keyspaces or keyspace/shards that this instance will monitor and repair. Defaults to all clusters in the topology. Example: "ks1,ks2/-80" + --config string config file name + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + --consul_auth_static_file string JSON File to read the topos/tokens from. + --emit_stats If set, emit stats to push-based monitoring and stats backends + --grpc_auth_static_client_creds string When using grpc_static_auth in the server, this file provides the credentials to use to authenticate with server. + --grpc_compression string Which protocol to use for compressing gRPC. Default: nothing. Supported: snappy + --grpc_enable_tracing Enable gRPC tracing. + --grpc_initial_conn_window_size int gRPC initial connection window size + --grpc_initial_window_size int gRPC initial window size + --grpc_keepalive_time duration After a duration of this time, if the client doesn't see any activity, it pings the server to see if the transport is still alive. (default 10s) + --grpc_keepalive_timeout duration After having pinged for keepalive check, the client waits for a duration of Timeout and if no activity is seen even after that the connection is closed. (default 10s) + --grpc_max_message_size int Maximum allowed RPC message size. Larger messages will be rejected by gRPC with the error 'exceeding the max size'. (default 16777216) + --grpc_prometheus Enable gRPC monitoring with Prometheus. + -h, --help display usage and exit + --instance-poll-time duration Timer duration on which VTOrc refreshes MySQL information (default 5s) + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --lameduck-period duration keep running at least this long after SIGTERM before stopping (default 50ms) + --lock-timeout duration Maximum time for which a shard/keyspace lock can be acquired for (default 45s) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --max-stack-size int configure the maximum stack size in bytes (default 67108864) + --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) + --onterm_timeout duration wait no more than this for OnTermSync handlers before stopping (default 10s) + --pid_file string If set, the process will write its pid to the named file, and delete it on graceful shutdown. + --port int port for the server + --pprof strings enable profiling + --prevent-cross-cell-failover Prevent VTOrc from promoting a primary in a different cell than the current primary in case of a failover + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --reasonable-replication-lag duration Maximum replication lag on replicas which is deemed to be acceptable (default 10s) + --recovery-period-block-duration duration Duration for which a new recovery is blocked on an instance after running a recovery (default 30s) + --recovery-poll-duration duration Timer duration on which VTOrc polls its database to run a recovery (default 1s) + --remote_operation_timeout duration time to wait for a remote operation (default 15s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --shutdown_wait_time duration Maximum time to wait for VTOrc to release all the locks that it is holding before shutting down on SIGTERM (default 30s) + --snapshot-topology-interval duration Timer duration on which VTOrc takes a snapshot of the current MySQL information it has in the database. Should be in multiple of hours + --sqlite-data-file string SQLite Datafile to use as VTOrc's database (default "file::memory:?mode=memory&cache=shared") + --stats_backend string The name of the registered push-based monitoring/stats backend to use + --stats_combine_dimensions string List of dimensions to be combined into a single "all" value in exported stats vars + --stats_common_tags strings Comma-separated list of common tags for the stats backend. It provides both label and values. Example: label1:value1,label2:value2 + --stats_drop_variables string Variables to be dropped from the list of exported variables. + --stats_emit_period duration Interval between emitting stats to all registered backends (default 1m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class + --tablet_manager_grpc_ca string the server ca to use to validate servers when connecting + --tablet_manager_grpc_cert string the cert to use to connect + --tablet_manager_grpc_concurrency int concurrency to use to talk to a vttablet server for performance-sensitive RPCs (like ExecuteFetchAs{Dba,AllPrivs,App}) (default 8) + --tablet_manager_grpc_connpool_size int number of tablets to keep tmclient connections open to (default 100) + --tablet_manager_grpc_crl string the server crl to use to validate server certificates when connecting + --tablet_manager_grpc_key string the key to use to connect + --tablet_manager_grpc_server_name string the server name to use to validate server certificate + --tablet_manager_protocol string Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc") + --topo-information-refresh-duration duration Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s) + --topo_consul_lock_delay duration LockDelay for consul session. (default 15s) + --topo_consul_lock_session_checks string List of checks for consul session. (default "serfHealth") + --topo_consul_lock_session_ttl string TTL for consul session. + --topo_consul_watch_poll_duration duration time of the long poll for watch queries. (default 30s) + --topo_etcd_lease_ttl int Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30) + --topo_etcd_tls_ca string path to the ca to use to validate the server cert when connecting to the etcd topo server + --topo_etcd_tls_cert string path to the client cert to use to connect to the etcd topo server, requires topo_etcd_tls_key, enables TLS + --topo_etcd_tls_key string path to the client key to use to connect to the etcd topo server, enables TLS + --topo_global_root string the path of the global topology data in the global topology server + --topo_global_server_address string the address of the global topology server + --topo_implementation string the topology implementation to use + --topo_k8s_context string The kubeconfig context to use, overrides the 'current-context' from the config + --topo_k8s_kubeconfig string Path to a valid kubeconfig file. When running as a k8s pod inside the same cluster you wish to use as the topo, you may omit this and the below arguments, and Vitess is capable of auto-discovering the correct values. https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod + --topo_k8s_namespace string The kubernetes namespace to use for all objects. Default comes from the context or in-cluster config + --topo_zk_auth_file string auth to use when connecting to the zk topo server, file contents should be :, e.g., digest:user:pass + --topo_zk_base_timeout duration zk base timeout (see zk.Connect) (default 30s) + --topo_zk_max_concurrency int maximum number of pending requests to send to a Zookeeper server. (default 64) + --topo_zk_tls_ca string the server ca to use to validate servers when connecting to the zk topo server + --topo_zk_tls_cert string the cert to use to connect to the zk topo server, requires topo_zk_tls_key, enables TLS + --topo_zk_tls_key string the key to use to connect to the zk topo server, enables TLS + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --wait-replicas-timeout duration Duration for which to wait for replica's to respond when issuing RPCs (default 30s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index a0577bad142..d8956aea341 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -34,6 +34,12 @@ Usage of vttablet: --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") --compression-engine-name string compressor engine used for compression. (default "pargzip") --compression-level int what level to pass to the compressor. (default 1) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consolidator-stream-query-size int Configure the stream consolidator query size in bytes. Setting to 0 disables the stream consolidator. (default 2097152) --consolidator-stream-total-size int Configure the stream consolidator total size in bytes. Setting to 0 disables the stream consolidator. (default 134217728) --consul_auth_static_file string JSON File to read the topos/tokens from. diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index be553107af0..4254f58c398 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -15,6 +15,12 @@ Usage of vttestserver: --charset string MySQL charset (default "utf8mb4") --compression-engine-name string compressor engine used for compression. (default "pargzip") --compression-level int what level to pass to the compressor. (default 1) + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). --consul_auth_static_file string JSON File to read the topos/tokens from. --data_dir string Directory where the data files will be placed, defaults to a random directory under /vt/vtdataroot --dba_idle_timeout duration Idle timeout for dba connections (default 1m0s) diff --git a/go/flags/endtoend/zkctl.txt b/go/flags/endtoend/zkctl.txt index e7e41c4cb4d..f52a5f7fe1b 100644 --- a/go/flags/endtoend/zkctl.txt +++ b/go/flags/endtoend/zkctl.txt @@ -1,18 +1,24 @@ Usage of zkctl: - --alsologtostderr log to standard error as well as files - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") - --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") + --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname diff --git a/go/flags/endtoend/zkctld.txt b/go/flags/endtoend/zkctld.txt index 6ec026be814..e2187ead03f 100644 --- a/go/flags/endtoend/zkctld.txt +++ b/go/flags/endtoend/zkctld.txt @@ -1,19 +1,25 @@ Usage of zkctld: - --alsologtostderr log to standard error as well as files - -h, --help display usage and exit - --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) - --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) - --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) - --log_dir string If non-empty, write log files in this directory - --log_err_stacks log stack traces for errors - --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) - --logtostderr log to standard error instead of files - --pprof strings enable profiling - --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) - --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) - --stderrthreshold severity logs at or above this threshold go to stderr (default 1) - --v Level log level for V logs - -v, --version print binary version - --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging - --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") - --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname + --alsologtostderr log to standard error as well as files + --config-file string Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored. + --config-file-not-found-handling ConfigFileNotFoundHandling Behavior when a config file is not found. (Options: error, exit, ignore, warn) (default warn) + --config-name string Name of the config file (without extension) to search for. (default "vtconfig") + --config-path strings Paths to search for config files in. (default [{{.Workdir}}]) + --config-persistence-min-interval duration minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done). (default 1s) + --config-type string Config file type (omit to infer config type from file extension). + -h, --help display usage and exit + --keep_logs duration keep logs for this long (using ctime) (zero to keep forever) + --keep_logs_by_mtime duration keep logs for this long (using mtime) (zero to keep forever) + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_err_stacks log stack traces for errors + --log_rotate_max_size uint size in bytes at which logs are rotated (glog.MaxSize) (default 1887436800) + --logtostderr log to standard error instead of files + --pprof strings enable profiling + --purge_logs_interval duration how often try to remove old logs (default 1h0m0s) + --security_policy string the name of a registered security policy to use for controlling access to URLs - empty means allow all for anyone (built-in policies: deny-all, read-only) + --stderrthreshold severity logs at or above this threshold go to stderr (default 1) + --v Level log level for V logs + -v, --version print binary version + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging + --zk.cfg string zkid@server1:leaderPort1:electionPort1:clientPort1,...) (default "6@:3801:3802:3803") + --zk.myid uint which server do you want to be? only needed when running multiple instance on one box, otherwise myid is implied by hostname diff --git a/go/flagutil/enum.go b/go/flagutil/enum.go new file mode 100644 index 00000000000..5bc279ee493 --- /dev/null +++ b/go/flagutil/enum.go @@ -0,0 +1,117 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flagutil + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +// StringEnum provides a string-like flag value that raises an error if given a +// value not in the set of allowed choices. +// +// This parse-time validation can be case-sensitive or not, depending on which +// constructor (NewStringEnum vs NewCaseInsensitiveStringEnum) was used. +type StringEnum struct { + name string + val string + + caseInsensitive bool + choices map[string]struct{} + choiceNames []string + choiceMapper func(string) string +} + +// ErrInvalidChoice is returned when parsing a value that is not a valid choice +// for the StringEnum flag. +var ErrInvalidChoice = errors.New("invalid choice for enum") + +// NewStringEnum returns a new string enum flag with the given name, default, +// and choices. +// +// Parse-time validation is case-sensitive. +func NewStringEnum(name string, initialValue string, choices []string) *StringEnum { + return newStringEnum(name, initialValue, choices, false) +} + +// NewCaseInsensitiveStringEnum returns a new string enum flag with the given +// name, default, and choices. +// +// Parse-time validation is case-insensitive. +func NewCaseInsensitiveStringEnum(name string, initialValue string, choices []string) *StringEnum { + return newStringEnum(name, initialValue, choices, true) +} + +func newStringEnum(name string, initialValue string, choices []string, caseInsensitive bool) *StringEnum { + choiceMapper := func(s string) string { return s } + choiceMap := map[string]struct{}{} + + if caseInsensitive { + choiceMapper = strings.ToLower + } + + for _, choice := range choices { + choiceMap[choiceMapper(choice)] = struct{}{} + } + + choiceNames := make([]string, 0, len(choiceMap)) + for choice := range choiceMap { + choiceNames = append(choiceNames, choice) + } + sort.Strings(choiceNames) + + if initialValue != "" { + if _, ok := choiceMap[choiceMapper(initialValue)]; !ok { + // This will panic if we've misconfigured something in the source code. + // It's not a user-error, so it had damn-well be better caught by a test + // somewhere. + panic(fmt.Errorf("%w (valid choices: %v)", ErrInvalidChoice, choiceNames)) + } + } + + return &StringEnum{ + name: name, + val: initialValue, + choices: choiceMap, + choiceNames: choiceNames, + choiceMapper: choiceMapper, + } +} + +// Set is part of the pflag.Value interface. +func (s *StringEnum) Set(arg string) error { + if _, ok := s.choices[s.choiceMapper(arg)]; !ok { + msg := "%w (valid choices: %v" + if s.caseInsensitive { + msg += " [case insensitive]" + } + msg += ")" + return fmt.Errorf(msg, ErrInvalidChoice, s.choiceNames) + } + + s.val = arg + + return nil +} + +// String is part of the pflag.Value interface. +func (s *StringEnum) String() string { return s.val } + +// Type is part of the pflag.Value interface. +func (s *StringEnum) Type() string { return "string" } diff --git a/go/test/endtoend/cluster/cluster_util.go b/go/test/endtoend/cluster/cluster_util.go index 59327ff6503..07a9e09df45 100644 --- a/go/test/endtoend/cluster/cluster_util.go +++ b/go/test/endtoend/cluster/cluster_util.go @@ -181,12 +181,22 @@ func getTablet(tabletGrpcPort int, hostname string) *topodatapb.Tablet { func filterResultForWarning(input string) string { lines := strings.Split(input, "\n") var result string - for _, line := range lines { + for i, line := range lines { if strings.Contains(line, "WARNING: vtctl should only be used for VDiff v1 workflows. Please use VDiff v2 and consider using vtctldclient for all other commands.") { continue } - result = result + line + "\n" + + if strings.Contains(line, "Failed to read in config") && strings.Contains(line, `Config File "vtconfig" Not Found in`) { + continue + } + + result += line + + if i < len(lines)-1 { + result += "\n" + } } + return result } diff --git a/go/test/endtoend/cluster/vtctlclient_process.go b/go/test/endtoend/cluster/vtctlclient_process.go index 3c4941da2c2..cf75ec1a5fa 100644 --- a/go/test/endtoend/cluster/vtctlclient_process.go +++ b/go/test/endtoend/cluster/vtctlclient_process.go @@ -215,7 +215,7 @@ func (vtctlclient *VtctlClientProcess) ExecuteCommandWithOutput(args ...string) } time.Sleep(retryDelay) } - return filterResultWhenRunsForCoverage(resultStr), err + return filterResultForWarning(filterResultWhenRunsForCoverage(resultStr)), err } // VtctlClientProcessInstance returns a VtctlProcess handle for vtctlclient process diff --git a/go/trace/plugin_datadog.go b/go/trace/plugin_datadog.go index 25f0b6894cd..b101607592d 100644 --- a/go/trace/plugin_datadog.go +++ b/go/trace/plugin_datadog.go @@ -3,40 +3,58 @@ package trace import ( "fmt" "io" + "net" "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentracer" ddtracer "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + + "vitess.io/vitess/go/viperutil" ) var ( - dataDogHost string - dataDogPort string + dataDogConfigKey = viperutil.KeyPrefixFunc(configKey("datadog")) + + dataDogHost = viperutil.Configure( + dataDogConfigKey("agent.host"), + viperutil.Options[string]{ + FlagName: "datadog-agent-host", + }, + ) + dataDogPort = viperutil.Configure( + dataDogConfigKey("agent.port"), + viperutil.Options[string]{ + FlagName: "datadog-agent-port", + }, + ) ) func init() { // If compiled with plugin_datadaog, ensure that trace.RegisterFlags // includes datadaog tracing flags. pluginFlags = append(pluginFlags, func(fs *pflag.FlagSet) { - fs.StringVar(&dataDogHost, "datadog-agent-host", "", "host to send spans to. if empty, no tracing will be done") - fs.StringVar(&dataDogPort, "datadog-agent-port", "", "port to send spans to. if empty, no tracing will be done") + fs.String("datadog-agent-host", "", "host to send spans to. if empty, no tracing will be done") + fs.String("datadog-agent-port", "", "port to send spans to. if empty, no tracing will be done") + + viperutil.BindFlags(fs, dataDogHost, dataDogPort) }) } func newDatadogTracer(serviceName string) (tracingService, io.Closer, error) { - if dataDogHost == "" || dataDogPort == "" { + host, port := dataDogHost.Get(), dataDogPort.Get() + if host == "" || port == "" { return nil, nil, fmt.Errorf("need host and port to datadog agent to use datadog tracing") } opts := []ddtracer.StartOption{ - ddtracer.WithAgentAddr(dataDogHost + ":" + dataDogPort), + ddtracer.WithAgentAddr(net.JoinHostPort(host, port)), ddtracer.WithServiceName(serviceName), ddtracer.WithDebugMode(true), ddtracer.WithSampler(ddtracer.NewRateSampler(samplingRate.Get())), } - if enableLogging { + if enableLogging.Get() { opts = append(opts, ddtracer.WithLogger(&traceLogger{})) } diff --git a/go/trace/plugin_jaeger.go b/go/trace/plugin_jaeger.go index b1449bc576f..9da9546c2ff 100644 --- a/go/trace/plugin_jaeger.go +++ b/go/trace/plugin_jaeger.go @@ -18,14 +18,12 @@ package trace import ( "io" - "os" "github.com/opentracing/opentracing-go" "github.com/spf13/pflag" - "github.com/uber/jaeger-client-go" "github.com/uber/jaeger-client-go/config" - "vitess.io/vitess/go/flagutil" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" ) @@ -36,18 +34,40 @@ included but nothing Jaeger specific. */ var ( - agentHost string - samplingType = flagutil.NewOptionalString("const") - samplingRate = flagutil.NewOptionalFloat64(0.1) + jaegerConfigKey = viperutil.KeyPrefixFunc(configKey("jaeger")) + agentHost = viperutil.Configure( + jaegerConfigKey("agent-host"), + viperutil.Options[string]{ + FlagName: "jaeger-agent-host", + }, + ) + samplingType = viperutil.Configure( + jaegerConfigKey("sampling_type"), + viperutil.Options[string]{ + Default: "const", + EnvVars: []string{"JAEGER_SAMPLER_TYPE"}, + FlagName: "tracing-sampling-type", + }, + ) + samplingRate = viperutil.Configure( + jaegerConfigKey("sampling_rate"), + viperutil.Options[float64]{ + Default: 0.1, + EnvVars: []string{"JAEGER_SAMPLER_PARAM"}, + FlagName: "tracing-sampling-rate", + }, + ) ) func init() { // If compiled with plugin_jaeger, ensure that trace.RegisterFlags includes // jaeger tracing flags. pluginFlags = append(pluginFlags, func(fs *pflag.FlagSet) { - fs.StringVar(&agentHost, "jaeger-agent-host", "", "host and port to send spans to. if empty, no tracing will be done") - fs.Var(samplingType, "tracing-sampling-type", "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") - fs.Var(samplingRate, "tracing-sampling-rate", "sampling rate for the probabilistic jaeger sampler") + fs.String("jaeger-agent-host", agentHost.Default(), "host and port to send spans to. if empty, no tracing will be done") + fs.String("tracing-sampling-type", samplingType.Default(), "sampling strategy to use for jaeger. possible values are 'const', 'probabilistic', 'rateLimiting', or 'remote'") + fs.Float64("tracing-sampling-rate", samplingRate.Default(), "sampling rate for the probabilistic jaeger sampler") + + viperutil.BindFlags(fs, agentHost, samplingRate, samplingType) }) } @@ -79,32 +99,17 @@ func newJagerTracerFromEnv(serviceName string) (tracingService, io.Closer, error } // Allow command line args to override environment variables. - if agentHost != "" { - cfg.Reporter.LocalAgentHostPort = agentHost + if host := agentHost.Get(); host != "" { + cfg.Reporter.LocalAgentHostPort = host } log.Infof("Tracing to: %v as %v", cfg.Reporter.LocalAgentHostPort, cfg.ServiceName) - if os.Getenv("JAEGER_SAMPLER_PARAM") == "" { - // If the environment variable was not set, we take the flag regardless - // of whether it was explicitly set on the command line. - cfg.Sampler.Param = samplingRate.Get() - } else if samplingRate.IsSet() { - // If the environment variable was set, but the user also explicitly - // passed the command line flag, the flag takes precedence. - cfg.Sampler.Param = samplingRate.Get() - } - - if samplingType.IsSet() { - cfg.Sampler.Type = samplingType.Get() - } else if cfg.Sampler.Type == "" { - log.Infof("--tracing-sampler-type was not set, and JAEGER_SAMPLER_TYPE was not set, defaulting to const sampler") - cfg.Sampler.Type = jaeger.SamplerTypeConst - } - + cfg.Sampler.Param = samplingRate.Get() + cfg.Sampler.Type = samplingType.Get() log.Infof("Tracing sampler type %v (param: %v)", cfg.Sampler.Type, cfg.Sampler.Param) var opts []config.Option - if enableLogging { + if enableLogging.Get() { opts = append(opts, config.Logger(&traceLogger{})) } else if cfg.Reporter.LogSpans { log.Warningf("JAEGER_REPORTER_LOG_SPANS was set, but --tracing-enable-logging was not; spans will not be logged") diff --git a/go/trace/trace.go b/go/trace/trace.go index dd0e1b56b62..7c43b4afedc 100644 --- a/go/trace/trace.go +++ b/go/trace/trace.go @@ -28,6 +28,7 @@ import ( "github.com/spf13/pflag" "google.golang.org/grpc" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" ) @@ -125,6 +126,8 @@ type tracingService interface { // object to make sure that all spans are sent to the backend before the process exits. type TracerFactory func(serviceName string) (tracingService, io.Closer, error) +const configKeyPrefix = "trace" + var ( // tracingBackendFactories should be added to by a plugin during init() to install itself tracingBackendFactories = make(map[string]TracerFactory) @@ -133,15 +136,30 @@ var ( /* flags */ - tracingServer = "noop" - enableLogging bool + configKey = viperutil.KeyPrefixFunc(configKeyPrefix) + + tracingServer = viperutil.Configure( + configKey("service"), + viperutil.Options[string]{ + Default: "noop", + FlagName: "tracer", + }, + ) + enableLogging = viperutil.Configure( + configKey("enable-logging"), + viperutil.Options[bool]{ + FlagName: "tracing-enable-logging", + }, + ) pluginFlags []func(fs *pflag.FlagSet) ) func RegisterFlags(fs *pflag.FlagSet) { - fs.StringVar(&tracingServer, "tracer", "noop", "tracing service to use") - fs.BoolVar(&enableLogging, "tracing-enable-logging", false, "whether to enable logging in the tracing service") + fs.String("tracer", tracingServer.Default(), "tracing service to use") + fs.Bool("tracing-enable-logging", false, "whether to enable logging in the tracing service") + + viperutil.BindFlags(fs, tracingServer, enableLogging) for _, fn := range pluginFlags { fn(fs) @@ -150,20 +168,21 @@ func RegisterFlags(fs *pflag.FlagSet) { // StartTracing enables tracing for a named service func StartTracing(serviceName string) io.Closer { - factory, ok := tracingBackendFactories[tracingServer] + tracingBackend := tracingServer.Get() + factory, ok := tracingBackendFactories[tracingBackend] if !ok { return fail(serviceName) } tracer, closer, err := factory(serviceName) if err != nil { - log.Error(vterrors.Wrapf(err, "failed to create a %s tracer", tracingServer)) + log.Error(vterrors.Wrapf(err, "failed to create a %s tracer", tracingBackend)) return &nilCloser{} } currentTracer = tracer - if tracingServer != "noop" { - log.Infof("successfully started tracing with [%s]", tracingServer) + if tracingBackend != "noop" { + log.Infof("successfully started tracing with [%s]", tracingBackend) } return closer diff --git a/go/trace/trace_test.go b/go/trace/trace_test.go index 35500c93b4e..08027a35c85 100644 --- a/go/trace/trace_test.go +++ b/go/trace/trace_test.go @@ -23,7 +23,10 @@ import ( "strings" "testing" + "github.com/spf13/viper" "google.golang.org/grpc" + + "vitess.io/vitess/go/viperutil/vipertest" ) func TestFakeSpan(t *testing.T) { @@ -49,13 +52,16 @@ func TestRegisterService(t *testing.T) { return tracer, tracer, nil } - tracingServer = fakeName + v := viper.New() + t.Cleanup(vipertest.Stub(t, v, tracingServer)) + + v.Set(tracingServer.Key(), fakeName) serviceName := "vtservice" closer := StartTracing(serviceName) tracer, ok := closer.(*fakeTracer) if !ok { - t.Fatalf("did not get the expected tracer") + t.Fatalf("did not get the expected tracer, got %+v (%T)", tracer, tracer) } if tracer.name != serviceName { diff --git a/go/viperutil/config.go b/go/viperutil/config.go new file mode 100644 index 00000000000..2af1f0c5419 --- /dev/null +++ b/go/viperutil/config.go @@ -0,0 +1,297 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "context" + "fmt" + "os" + "reflect" + "sort" + "strings" + "time" + + "github.com/mitchellh/mapstructure" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/funcs" + "vitess.io/vitess/go/viperutil/internal/log" + "vitess.io/vitess/go/viperutil/internal/registry" + "vitess.io/vitess/go/viperutil/internal/value" +) + +var ( + configPaths = Configure( + "config.paths", + Options[[]string]{ + GetFunc: funcs.GetPath, + EnvVars: []string{"VT_CONFIG_PATH"}, + FlagName: "config-path", + }, + ) + configType = Configure( + "config.type", + Options[string]{ + EnvVars: []string{"VT_CONFIG_TYPE"}, + FlagName: "config-type", + }, + ) + configName = Configure( + "config.name", + Options[string]{ + Default: "vtconfig", + EnvVars: []string{"VT_CONFIG_NAME"}, + FlagName: "config-name", + }, + ) + configFile = Configure( + "config.file", + Options[string]{ + EnvVars: []string{"VT_CONFIG_FILE"}, + FlagName: "config-file", + }, + ) + configFileNotFoundHandling = Configure( + "config.notfound.handling", + Options[ConfigFileNotFoundHandling]{ + Default: WarnOnConfigFileNotFound, + GetFunc: getHandlingValue, + }, + ) + configPersistenceMinInterval = Configure( + "config.persistence.min_interval", + Options[time.Duration]{ + Default: time.Second, + EnvVars: []string{"VT_CONFIG_PERSISTENCE_MIN_INTERVAL"}, + FlagName: "config-persistence-min-interval", + }, + ) +) + +func init() { + wd, err := os.Getwd() + if err != nil { + log.WARN("failed to get working directory (err=%v), not appending to default config-paths", err) + return + } + + configPaths.(*value.Static[[]string]).DefaultVal = []string{wd} + // Need to re-trigger the SetDefault call done during Configure. + registry.Static.SetDefault(configPaths.Key(), configPaths.Default()) +} + +// RegisterFlags installs the flags that control viper config-loading behavior. +// It is exported to be called by servenv before parsing flags for all binaries. +// +// It cannot be registered here via servenv.OnParse since this causes an import +// cycle. +func RegisterFlags(fs *pflag.FlagSet) { + fs.StringSlice("config-path", configPaths.Default(), "Paths to search for config files in.") + fs.String("config-type", configType.Default(), "Config file type (omit to infer config type from file extension).") + fs.String("config-name", configName.Default(), "Name of the config file (without extension) to search for.") + fs.String("config-file", configFile.Default(), "Full path of the config file (with extension) to use. If set, --config-path, --config-type, and --config-name are ignored.") + fs.Duration("config-persistence-min-interval", configPersistenceMinInterval.Default(), "minimum interval between persisting dynamic config changes back to disk (if no change has occurred, nothing is done).") + + var h = configFileNotFoundHandling.Default() + fs.Var(&h, "config-file-not-found-handling", fmt.Sprintf("Behavior when a config file is not found. (Options: %s)", strings.Join(handlingNames, ", "))) + + BindFlags(fs, configPaths, configType, configName, configFile) +} + +// LoadConfig attempts to find, and then load, a config file for viper-backed +// config values to use. +// +// Config searching follows the behavior used by viper [1], namely: +// - --config-file (full path, including extension) if set will be used to the +// exclusion of all other flags. +// - --config-type is required if the config file does not have one of viper's +// supported extensions (.yaml, .yml, .json, and so on) +// +// An additional --config-file-not-found-handling flag controls how to treat the +// situation where viper cannot find any config files in any of the provided +// paths (for ex, users may want to exit immediately if a config file that +// should exist doesn't for some reason, or may wish to operate with flags and +// environment variables alone, and not use config files at all). +// +// If a config file is successfully loaded, then the dynamic registry will also +// start watching that file for changes. In addition, in-memory changes to the +// config (for example, from a vtgate or vttablet's debugenv) will be persisted +// back to disk, with writes occuring no more frequently than the +// --config-persistence-min-interval flag. +// +// A cancel function is returned to stop the re-persistence background thread, +// if one was started. +// +// [1]: https://github.com/spf13/viper#reading-config-files. +func LoadConfig() (context.CancelFunc, error) { + var err error + switch file := configFile.Get(); file { + case "": + if name := configName.Get(); name != "" { + registry.Static.SetConfigName(name) + + for _, path := range configPaths.Get() { + registry.Static.AddConfigPath(path) + } + + if cfgType := configType.Get(); cfgType != "" { + registry.Static.SetConfigType(cfgType) + } + + err = registry.Static.ReadInConfig() + } + default: + registry.Static.SetConfigFile(file) + err = registry.Static.ReadInConfig() + } + + if err != nil { + if nferr, ok := err.(viper.ConfigFileNotFoundError); ok { + msg := "Failed to read in config %s: %s" + switch configFileNotFoundHandling.Get() { + case WarnOnConfigFileNotFound: + log.WARN(msg, registry.Static.ConfigFileUsed(), nferr.Error()) + fallthrough // after warning, ignore the error + case IgnoreConfigFileNotFound: + err = nil + case ErrorOnConfigFileNotFound: + log.ERROR(msg, registry.Static.ConfigFileUsed(), nferr.Error()) + case ExitOnConfigFileNotFound: + log.CRITICAL(msg, registry.Static.ConfigFileUsed(), nferr.Error()) + } + } + } + + if err != nil { + return nil, err + } + + return registry.Dynamic.Watch(context.Background(), registry.Static, configPersistenceMinInterval.Get()) +} + +// NotifyConfigReload adds a subscription that the dynamic registry will attempt +// to notify on config changes. The notification fires after the updated config +// has been loaded from disk into the live config. +// +// Analogous to signal.Notify, notifications are sent non-blocking, so users +// should account for this when writing code to consume from the channel. +// +// This function must be called prior to LoadConfig; it will panic if called +// after the dynamic registry has started watching the loaded config. +func NotifyConfigReload(ch chan<- struct{}) { + registry.Dynamic.Notify(ch) +} + +// ConfigFileNotFoundHandling is an enum to control how LoadConfig treats errors +// of type viper.ConfigFileNotFoundError when loading a config. +type ConfigFileNotFoundHandling int + +const ( + // IgnoreConfigFileNotFound causes LoadConfig to completely ignore a + // ConfigFileNotFoundError (i.e. not even logging it). + IgnoreConfigFileNotFound ConfigFileNotFoundHandling = iota + // WarnOnConfigFileNotFound causes LoadConfig to log a warning with details + // about the failed config load, but otherwise proceeds with the given + // process, which will get config values entirely from defaults, + // envirnoment variables, and flags. + WarnOnConfigFileNotFound + // ErrorOnConfigFileNotFound causes LoadConfig to return the + // ConfigFileNotFoundError after logging an error. + ErrorOnConfigFileNotFound + // ExitOnConfigFileNotFound causes LoadConfig to log.Fatal on a + // ConfigFileNotFoundError. + ExitOnConfigFileNotFound +) + +var ( + handlingNames []string + handlingNamesToValues = map[string]int{ + "ignore": int(IgnoreConfigFileNotFound), + "warn": int(WarnOnConfigFileNotFound), + "error": int(ErrorOnConfigFileNotFound), + "exit": int(ExitOnConfigFileNotFound), + } + handlingValuesToNames map[int]string +) + +func getHandlingValue(v *viper.Viper) func(key string) ConfigFileNotFoundHandling { + return func(key string) (h ConfigFileNotFoundHandling) { + if err := v.UnmarshalKey(key, &h, viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(decodeHandlingValue))); err != nil { + h = IgnoreConfigFileNotFound + log.WARN("failed to unmarshal %s: %s; defaulting to %s", err.Error(), key, err.Error(), h.String()) + } + + return h + } +} + +func decodeHandlingValue(from, to reflect.Type, data any) (any, error) { + var h ConfigFileNotFoundHandling + if to != reflect.TypeOf(h) { + return data, nil + } + + switch { + case from == reflect.TypeOf(h): + return data.(ConfigFileNotFoundHandling), nil + case from.Kind() == reflect.Int: + return ConfigFileNotFoundHandling(data.(int)), nil + case from.Kind() == reflect.String: + if err := h.Set(data.(string)); err != nil { + return h, err + } + + return h, nil + } + + return data, fmt.Errorf("invalid value for ConfigHandlingType: %v", data) +} + +func init() { + handlingNames = make([]string, 0, len(handlingNamesToValues)) + handlingValuesToNames = make(map[int]string, len(handlingNamesToValues)) + + for name, val := range handlingNamesToValues { + handlingValuesToNames[val] = name + handlingNames = append(handlingNames, name) + } + + sort.Slice(handlingNames, func(i, j int) bool { + return handlingNames[i] < handlingNames[j] + }) +} + +func (h *ConfigFileNotFoundHandling) Set(arg string) error { + larg := strings.ToLower(arg) + if v, ok := handlingNamesToValues[larg]; ok { + *h = ConfigFileNotFoundHandling(v) + return nil + } + + return fmt.Errorf("unknown handling name %s", arg) +} + +func (h *ConfigFileNotFoundHandling) String() string { + if name, ok := handlingValuesToNames[int(*h)]; ok { + return name + } + + return "" +} + +func (h *ConfigFileNotFoundHandling) Type() string { return "ConfigFileNotFoundHandling" } diff --git a/go/viperutil/config_test.go b/go/viperutil/config_test.go new file mode 100644 index 00000000000..8e00a4700ac --- /dev/null +++ b/go/viperutil/config_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetConfigHandlingValue(t *testing.T) { + v := viper.New() + v.SetDefault("default", ExitOnConfigFileNotFound) + v.SetConfigType("yaml") + + cfg := ` +foo: 2 +bar: "2" # not valid, defaults to "ignore" (0) +baz: error +duration: 10h +` + err := v.ReadConfig(strings.NewReader(strings.NewReplacer("\t", " ").Replace(cfg))) + require.NoError(t, err) + + getHandlingValueFunc := getHandlingValue(v) + assert.Equal(t, ErrorOnConfigFileNotFound, getHandlingValueFunc("foo"), "failed to get int value") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("bar"), "failed to get int-like string value") + assert.Equal(t, ErrorOnConfigFileNotFound, getHandlingValueFunc("baz"), "failed to get string value") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("notset"), "failed to get value on unset key") + assert.Equal(t, IgnoreConfigFileNotFound, getHandlingValueFunc("duration"), "failed to get value on duration key") + assert.Equal(t, ExitOnConfigFileNotFound, getHandlingValueFunc("default"), "failed to get value on default key") +} diff --git a/go/viperutil/debug/debug.go b/go/viperutil/debug/debug.go new file mode 100644 index 00000000000..66cbc7f2962 --- /dev/null +++ b/go/viperutil/debug/debug.go @@ -0,0 +1,27 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "vitess.io/vitess/go/viperutil/internal/registry" +) + +// Debug provides the Debug functionality normally accessible to a given viper +// instance, but for a combination of the private static and dynamic registries. +func Debug() { + registry.Combined().Debug() +} diff --git a/go/viperutil/debug/handler.go b/go/viperutil/debug/handler.go new file mode 100644 index 00000000000..07442dd13ab --- /dev/null +++ b/go/viperutil/debug/handler.go @@ -0,0 +1,82 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package debug + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/spf13/viper" + + "vitess.io/vitess/go/acl" + "vitess.io/vitess/go/slices2" + "vitess.io/vitess/go/viperutil/internal/registry" +) + +// HandlerFunc provides an http.HandlerFunc that renders the combined config +// registry (both static and dynamic) for debugging purposes. +// +// By default, this writes the config in viper's "debug" format (what you get +// if you call viper.Debug()). If the query parameter "format" is present, and +// matches one of viper's supported config extensions (case-insensitively), the +// combined config will be written to the response in that format. +// +// Example requests: +// - GET /debug/config +// - GET /debug/config?format=json +// - GET /debug/config?format=yaml +func HandlerFunc(w http.ResponseWriter, r *http.Request) { + if err := acl.CheckAccessHTTP(r, acl.DEBUGGING); err != nil { + acl.SendError(w, err) + return + } + + v := registry.Combined() + format := strings.ToLower(r.URL.Query().Get("format")) + switch { + case format == "": + v.DebugTo(w) + case slices2.Any(viper.SupportedExts, func(ext string) bool { return ext == format }): + // Got a supported format; write the config to a tempfile in that format, + // then copy it to the response. + // + // (Sadly, viper does not yet have a WriteConfigTo(w io.Writer), so we have + // to do this little hacky workaround). + v.SetConfigType(format) + tmp, err := os.CreateTemp("", "viper_debug") + if err != nil { + http.Error(w, fmt.Sprintf("failed to render config to tempfile: %v", err), http.StatusInternalServerError) + return + } + defer os.Remove(tmp.Name()) + + if err := v.WriteConfigAs(tmp.Name()); err != nil { + http.Error(w, fmt.Sprintf("failed to render config to tempfile: %v", err), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, tmp); err != nil { + http.Error(w, fmt.Sprintf("failed to write rendered config: %v", err), http.StatusInternalServerError) + return + } + default: + http.Error(w, fmt.Sprintf("unsupported config format %s", format), http.StatusBadRequest) + } +} diff --git a/go/viperutil/errors.go b/go/viperutil/errors.go new file mode 100644 index 00000000000..5d18774f998 --- /dev/null +++ b/go/viperutil/errors.go @@ -0,0 +1,33 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "vitess.io/vitess/go/viperutil/internal/sync" + "vitess.io/vitess/go/viperutil/internal/value" +) + +var ( + // ErrDuplicateWatch is returned when Watch is called multiple times on a + // single synced viper. Viper only supports reading/watching a single + // config file. + ErrDuplicateWatch = sync.ErrDuplicateWatch + // ErrNoFlagDefined is returned from Value's Flag method when the value was + // configured to bind to a given FlagName but the provided flag set does not + // define a flag with that name. + ErrNoFlagDefined = value.ErrNoFlagDefined +) diff --git a/go/viperutil/funcs/get.go b/go/viperutil/funcs/get.go new file mode 100644 index 00000000000..e33ffe9065f --- /dev/null +++ b/go/viperutil/funcs/get.go @@ -0,0 +1,37 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package funcs + +import ( + "strings" + + "github.com/spf13/viper" +) + +// GetPath returns a GetFunc that expands a slice of strings into individual +// paths based on standard POSIX shell $PATH separator parsing. +func GetPath(v *viper.Viper) func(key string) []string { + return func(key string) (paths []string) { + for _, val := range v.GetStringSlice(key) { + if val != "" { + paths = append(paths, strings.Split(val, ":")...) + } + } + + return paths + } +} diff --git a/go/viperutil/funcs/get_test.go b/go/viperutil/funcs/get_test.go new file mode 100644 index 00000000000..2af83e99aba --- /dev/null +++ b/go/viperutil/funcs/get_test.go @@ -0,0 +1,49 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package funcs_test + +import ( + "fmt" + + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/funcs" + "vitess.io/vitess/go/viperutil/internal/value" +) + +func ExampleGetPath() { + v := viper.New() + + val := viperutil.Configure("path", viperutil.Options[[]string]{ + GetFunc: funcs.GetPath, + }) + + stub(val, v) + + v.Set(val.Key(), []string{"/var/www", "/usr:/usr/bin", "/vt"}) + fmt.Println(val.Get()) + // Output: [/var/www /usr /usr/bin /vt] +} + +func stub[T any](val viperutil.Value[T], v *viper.Viper) { + // N.B.: You cannot do this in normal code because these types are internal + // to viperutil, but you also will not need to do this. However it's + // necessary for the example to work here. + base := val.(*value.Static[T]).Base + base.BoundGetFunc = base.GetFunc(v) +} diff --git a/go/viperutil/get_func.go b/go/viperutil/get_func.go new file mode 100644 index 00000000000..82a34df3fac --- /dev/null +++ b/go/viperutil/get_func.go @@ -0,0 +1,223 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "fmt" + "reflect" + "strconv" + "time" + + "github.com/spf13/viper" +) + +// GetFuncForType returns the default getter function for a given type T. A +// getter function is a function which takes a viper and returns a function that +// takes a key and (finally!) returns a value of type T. +// +// For example, the default getter for a value of type string is a function that +// takes a viper instance v and calls v.GetString with the provided key. +// +// In most cases, callers of Configure should be able to rely on the defaults +// provided here (and may refer to get_func_test.go for an up-to-date example +// of the provided functionalities), but if more fine-grained control is needed +// (this should be an **exceptional** circumstance), they may provide their own +// GetFunc as an option to Configure. +// +// This function may panic if called for an unsupported type. This is captured +// in the test code as well. +func GetFuncForType[T any]() func(v *viper.Viper) func(key string) T { + var ( + t T + f any + ) + + typ := reflect.TypeOf(t) + switch typ.Kind() { + case reflect.Bool: + f = func(v *viper.Viper) func(key string) bool { + return v.GetBool + } + case reflect.Int: + f = func(v *viper.Viper) func(key string) int { + return v.GetInt + } + case reflect.Int8: + f = getCastedInt[int8]() + case reflect.Int16: + f = getCastedInt[int16]() + case reflect.Int32: + f = func(v *viper.Viper) func(key string) int32 { + return v.GetInt32 + } + case reflect.Int64: + switch typ { + case reflect.TypeOf(time.Duration(0)): + f = func(v *viper.Viper) func(key string) time.Duration { + return v.GetDuration + } + default: + f = func(v *viper.Viper) func(key string) int64 { + return v.GetInt64 + } + } + case reflect.Uint: + f = func(v *viper.Viper) func(key string) uint { + return v.GetUint + } + case reflect.Uint8: + f = getCastedUint[uint8]() + case reflect.Uint16: + f = getCastedUint[uint16]() + case reflect.Uint32: + f = func(v *viper.Viper) func(key string) uint32 { + return v.GetUint32 + } + case reflect.Uint64: + f = func(v *viper.Viper) func(key string) uint64 { + return v.GetUint64 + } + case reflect.Uintptr: + // Unupported, fallthrough to `if f == nil` check below switch. + case reflect.Float32: + f = func(v *viper.Viper) func(key string) float32 { + return func(key string) float32 { + return float32(v.GetFloat64(key)) + } + } + case reflect.Float64: + f = func(v *viper.Viper) func(key string) float64 { + return v.GetFloat64 + } + case reflect.Complex64: + f = getComplex[complex64](64) + case reflect.Complex128: + f = getComplex[complex128](128) + case reflect.Array: + // Even though the code would be extremely similar to slice types, we + // cannot support arrays because there's no way to write a function that + // returns, say, [N]int, for some value of N which we only know at + // runtime. + panic("GetFuncForType does not support array types") + case reflect.Chan: + panic("GetFuncForType does not support channel types") + case reflect.Func: + panic("GetFuncForType does not support function types") + case reflect.Interface: + panic("GetFuncForType does not support interface types (specify a specific implementation type instead)") + case reflect.Map: + switch typ.Key().Kind() { + case reflect.String: + switch val := typ.Elem(); val.Kind() { + case reflect.String: + f = func(v *viper.Viper) func(key string) map[string]string { + return v.GetStringMapString + } + case reflect.Slice: + switch val.Elem().Kind() { + case reflect.String: + f = func(v *viper.Viper) func(key string) map[string][]string { + return v.GetStringMapStringSlice + } + } + case reflect.Interface: + f = func(v *viper.Viper) func(key string) map[string]interface{} { + return v.GetStringMap + } + } + } + case reflect.Pointer: + switch typ.Elem().Kind() { + case reflect.Struct: + f = unmarshalFunc[T]() + } + case reflect.Slice: + switch typ.Elem().Kind() { + case reflect.Int: + f = func(v *viper.Viper) func(key string) []int { + return v.GetIntSlice + } + case reflect.String: + f = func(v *viper.Viper) func(key string) []string { + return v.GetStringSlice + } + } + case reflect.String: + f = func(v *viper.Viper) func(key string) string { + return v.GetString + } + case reflect.Struct: + switch typ { + case reflect.TypeOf(time.Time{}): + f = func(v *viper.Viper) func(key string) time.Time { + return v.GetTime + } + default: + f2 := unmarshalFunc[*T]() + f = func(v *viper.Viper) func(key string) T { + getPointer := f2(v) + return func(key string) T { + return *(getPointer(key)) + } + } + } + } + + if f == nil { + panic(fmt.Sprintf("no default GetFunc for type %T; call Configure with a custom GetFunc", t)) + } + return f.(func(v *viper.Viper) func(key string) T) +} + +func unmarshalFunc[T any]() func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + t := new(T) + _ = v.UnmarshalKey(key, t) // TODO: panic on this error + return *t + } + } +} + +func getCastedInt[T int8 | int16]() func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + return T(v.GetInt(key)) + } + } +} + +func getCastedUint[T uint8 | uint16]() func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + return T(v.GetUint(key)) + } + } +} + +func getComplex[T complex64 | complex128](bitSize int) func(v *viper.Viper) func(key string) T { + return func(v *viper.Viper) func(key string) T { + return func(key string) T { + x, err := strconv.ParseComplex(v.GetString(key), bitSize) + if err != nil { + panic(err) // TODO: wrap with more details (key, type (64 vs 128), etc) + } + + return T(x) + } + } +} diff --git a/go/viperutil/get_func_test.go b/go/viperutil/get_func_test.go new file mode 100644 index 00000000000..346573704f1 --- /dev/null +++ b/go/viperutil/get_func_test.go @@ -0,0 +1,175 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +type myStruct struct { + Foo string + Bar int +} + +type myNestedStruct struct { + MyStruct *myStruct + Baz bool +} + +func TestGetFuncForType(t *testing.T) { + now := time.Now() + + v := viper.New() + v.Set("foo.bool", true) + v.Set("foo.int", 5) + v.Set("foo.duration", time.Second) + v.Set("foo.float", 5.1) + v.Set("foo.complex", fmt.Sprintf("%v", complex(1, 2))) + v.Set("foo.intslice", []int{1, 2, 3}) + v.Set("foo.stringslice", []string{"a", "b", "c"}) + v.Set("foo.string", "hello") + v.Set("foo.time", now) + + assert := assert.New(t) + + // Bool types + assert.Equal(true, get[bool](t, v, "foo.bool"), "GetFuncForType[bool](foo.bool)") + + // Int types + assert.Equal(5, get[int](t, v, "foo.int"), "GetFuncForType[int](foo.int)") + assert.Equal(int8(5), get[int8](t, v, "foo.int"), "GetFuncForType[int8](foo.int)") + assert.Equal(int16(5), get[int16](t, v, "foo.int"), "GetFuncForType[int16](foo.int)") + assert.Equal(int32(5), get[int32](t, v, "foo.int"), "GetFuncForType[int32](foo.int)") + assert.Equal(int64(5), get[int64](t, v, "foo.int"), "GetFuncForType[int64](foo.int)") + + // Duration types + assert.Equal(time.Second, get[time.Duration](t, v, "foo.duration"), "GetFuncForType[time.Duration](foo.duration)") + + // Uint types + assert.Equal(uint(5), get[uint](t, v, "foo.int"), "GetFuncForType[uint](foo.int)") + assert.Equal(uint8(5), get[uint8](t, v, "foo.int"), "GetFuncForType[uint8](foo.int)") + assert.Equal(uint16(5), get[uint16](t, v, "foo.int"), "GetFuncForType[uint16](foo.int)") + assert.Equal(uint32(5), get[uint32](t, v, "foo.int"), "GetFuncForType[uint32](foo.int)") + assert.Equal(uint64(5), get[uint64](t, v, "foo.int"), "GetFuncForType[uint64](foo.int)") + + // Float types + assert.Equal(5.1, get[float64](t, v, "foo.float"), "GetFuncForType[float64](foo.float)") + assert.Equal(float32(5.1), get[float32](t, v, "foo.float"), "GetFuncForType[float32](foo.float)") + assert.Equal(float64(5), get[float64](t, v, "foo.int"), "GetFuncForType[float64](foo.int)") + + // Complex types + assert.Equal(complex(5, 0), get[complex128](t, v, "foo.int"), "GetFuncForType[complex128](foo.int)") + assert.Equal(complex(5.1, 0), get[complex128](t, v, "foo.float"), "GetFuncForType[complex128](foo.float)") + assert.Equal(complex(1, 2), get[complex128](t, v, "foo.complex"), "GetFuncForType[complex128](foo.complex)") + assert.Equal(complex64(complex(5, 0)), get[complex64](t, v, "foo.int"), "GetFuncForType[complex64](foo.int)") + assert.Equal(complex64(complex(5.1, 0)), get[complex64](t, v, "foo.float"), "GetFuncForType[complex64](foo.float)") + assert.Equal(complex64(complex(1, 2)), get[complex64](t, v, "foo.complex"), "GetFuncForType[complex64](foo.complex)") + + // Slice types + assert.ElementsMatch([]int{1, 2, 3}, get[[]int](t, v, "foo.intslice"), "GetFuncForType[[]int](foo.intslice)") + assert.ElementsMatch([]string{"a", "b", "c"}, get[[]string](t, v, "foo.stringslice"), "GetFuncForType[[]string](foo.stringslice)") + + // String types + assert.Equal("hello", get[string](t, v, "foo.string"), "GetFuncForType[string](foo.string)") + + // Struct types + assert.Equal(now, get[time.Time](t, v, "foo.time"), "GetFuncForType[time.Time](foo.time)") + { + s := &myStruct{ + Foo: "hello", + Bar: 3, + } + v.Set("mystruct.foo", s.Foo) + v.Set("mystruct.bar", s.Bar) + + assert.Equal(s, get[*myStruct](t, v, "mystruct"), "GetFuncForType[*myStruct](mystruct)") + assert.IsType(&myStruct{}, get[*myStruct](t, v, "mystruct"), "GetFuncForType[*myStruct](mystruct) should return a pointer") + assert.Equal(*s, get[myStruct](t, v, "mystruct"), "GetFuncForType[myStruct](mystruct)") + assert.IsType(myStruct{}, get[myStruct](t, v, "mystruct"), "GetFuncForType[myStruct](mystruct) should return a struct (not pointer to struct)") + + s2 := &myNestedStruct{ + MyStruct: s, + Baz: true, + } + v.Set("mynestedstruct.mystruct.foo", s2.MyStruct.Foo) + v.Set("mynestedstruct.mystruct.bar", s2.MyStruct.Bar) + v.Set("mynestedstruct.baz", s2.Baz) + assert.Equal(*s2, get[myNestedStruct](t, v, "mynestedstruct"), "GetFuncForType[myNestedStruct](mynestedstruct)") + assert.IsType(myNestedStruct{}, get[myNestedStruct](t, v, "mynestedstruct"), "GetFuncForType[myNestedStruct](mynestedstruct) should return a struct (not pointer to struct)") + } + + // Map types. + v.Set("stringmap", map[string]string{ + "a": "A", + "b": "B", + }) + assert.Equal(map[string]string{"a": "A", "b": "B"}, get[map[string]string](t, v, "stringmap"), "GetFuncForType[map[string]string](stringmap)") + + v.Set("stringslicemap", map[string][]string{ + "uppers": strings.Split("ABCDEFG", ""), + "lowers": strings.Split("abcdefg", ""), + }) + assert.Equal(map[string][]string{"uppers": strings.Split("ABCDEFG", ""), "lowers": strings.Split("abcdefg", "")}, get[map[string][]string](t, v, "stringslicemap"), "GetFuncForType[map[string][]string](stringslicemap)") + + v.Set("anymap", map[string]any{ + "int": 5, + "bool": true, + "string": "hello", + }) + assert.Equal(map[string]any{"int": 5, "bool": true, "string": "hello"}, get[map[string]any](t, v, "anymap"), "GetFuncForType[map[string]any](anymap)") + + // Unsupported types. + t.Run("uintptr", func(t *testing.T) { + testPanic(t, GetFuncForType[uintptr], "GetFuncForType[uintptr]") + }) + t.Run("arrays", func(t *testing.T) { + testPanic(t, GetFuncForType[[5]int], "GetFuncForType[[5]int]") + testPanic(t, GetFuncForType[[3]string], "GetFuncForType[[3]string]") + }) + t.Run("channels", func(t *testing.T) { + testPanic(t, GetFuncForType[chan struct{}], "GetFuncForType[chan struct{}]") + testPanic(t, GetFuncForType[chan bool], "GetFuncForType[chan bool]") + testPanic(t, GetFuncForType[chan int], "GetFuncForType[chan int]") + testPanic(t, GetFuncForType[chan chan string], "GetFuncForType[chan chan string]") + }) + t.Run("funcs", func(t *testing.T) { + testPanic(t, GetFuncForType[func()], "GetFuncForType[func()]") + }) +} + +func testPanic[T any](t testing.TB, f func() func(v *viper.Viper) func(key string) T, fnName string) { + t.Helper() + + defer func() { + err := recover() + assert.NotNil(t, err, "%s should panic", fnName) + }() + + fn := f() + assert.Failf(t, fmt.Sprintf("%s should panic", fnName), "%s should panic; got %+v", fnName, fn) +} + +func get[T any](t testing.TB, v *viper.Viper, key string) T { + t.Helper() + return GetFuncForType[T]()(v)(key) +} diff --git a/go/viperutil/internal/log/log.go b/go/viperutil/internal/log/log.go new file mode 100644 index 00000000000..cded335cd47 --- /dev/null +++ b/go/viperutil/internal/log/log.go @@ -0,0 +1,57 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package log provides dual-use logging between vitess's vt/log package and +viper's jww log. +*/ +package log + +import ( + jww "github.com/spf13/jwalterweatherman" + + "vitess.io/vitess/go/vt/log" +) + +var ( + jwwlog = func(printer interface { + Printf(format string, args ...any) + }, vtlogger func(format string, args ...any)) func(format string, args ...any) { + switch vtlogger { + case nil: + return printer.Printf + default: + return func(format string, args ...any) { + printer.Printf(format, args...) + vtlogger(format, args...) + } + } + } + + // TRACE logs to viper's TRACE level, and nothing to vitess logs. + TRACE = jwwlog(jww.TRACE, nil) + // DEBUG logs to viper's DEBUG level, and nothing to vitess logs. + DEBUG = jwwlog(jww.DEBUG, nil) + // INFO logs to viper and vitess at INFO levels. + INFO = jwwlog(jww.INFO, log.Infof) + // WARN logs to viper and vitess at WARN/WARNING levels. + WARN = jwwlog(jww.WARN, log.Warningf) + // ERROR logs to viper and vitess at ERROR levels. + ERROR = jwwlog(jww.ERROR, log.Errorf) + // CRITICAL logs to viper at CRITICAL level, and then fatally logs to + // vitess, exiting the process. + CRITICAL = jwwlog(jww.CRITICAL, log.Fatalf) +) diff --git a/go/viperutil/internal/registry/registry.go b/go/viperutil/internal/registry/registry.go new file mode 100644 index 00000000000..10786a5f7b3 --- /dev/null +++ b/go/viperutil/internal/registry/registry.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/internal/sync" +) + +var ( + // Static is the registry for static config variables. These variables will + // never be affected by a Watch-ed config, and maintain their original + // values for the lifetime of the process. + Static = viper.New() + // Dynamic is the registry for dynamic config variables. If a config file is + // found by viper, it will be watched by a threadsafe wrapper around a + // second viper (see sync.Viper), and variables registered to it will pick + // up changes to that config file throughout the lifetime of the process. + Dynamic = sync.New() + + _ Bindable = (*viper.Viper)(nil) + _ Bindable = (*sync.Viper)(nil) +) + +// Bindable represents the methods needed to bind a value.Value to a given +// registry. It exists primarly to allow us to treat a sync.Viper as a +// viper.Viper for configuration registration purposes. +type Bindable interface { + BindEnv(vars ...string) error + BindPFlag(key string, flag *pflag.Flag) error + RegisterAlias(alias string, key string) + SetDefault(key string, value any) +} + +// Combined returns a viper combining the Static and Dynamic registries. +func Combined() *viper.Viper { + v := viper.New() + _ = v.MergeConfigMap(Static.AllSettings()) + _ = v.MergeConfigMap(Dynamic.AllSettings()) + + v.SetConfigFile(Static.ConfigFileUsed()) + return v +} diff --git a/go/viperutil/internal/sync/sync.go b/go/viperutil/internal/sync/sync.go new file mode 100644 index 00000000000..11cb028c286 --- /dev/null +++ b/go/viperutil/internal/sync/sync.go @@ -0,0 +1,308 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sync + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/internal/log" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" +) + +// Viper is a wrapper around a pair of viper.Viper instances to provide config- +// reloading in a threadsafe manner. +// +// It maintains one viper, called "disk", which does the actual config watch and +// reload (via viper's WatchConfig), and a second viper, called "live", which +// Values (registered via viperutil.Configure with Dynamic=true) access their +// settings from. The "live" config only updates after blocking all values from +// reading in order to swap in the most recently-loaded config from the "disk". +type Viper struct { + m sync.Mutex // prevents races between loadFromDisk and AllSettings + disk *viper.Viper + live *viper.Viper + keys map[string]*sync.RWMutex + + subscribers []chan<- struct{} + watchingConfig bool + + setCh chan struct{} +} + +// New returns a new synced Viper. +func New() *Viper { + return &Viper{ + disk: viper.New(), + live: viper.New(), + keys: map[string]*sync.RWMutex{}, + setCh: make(chan struct{}, 1), + } +} + +// Set sets the given key to the given value, in both the disk and live vipers. +func (v *Viper) Set(key string, value any) { + m, ok := v.keys[key] + if !ok { + return + } + + m.Lock() + defer m.Unlock() + + v.m.Lock() + defer v.m.Unlock() + + // We must not update v.disk here; explicit calls to Set will supercede all + // future config reloads. + v.live.Set(key, value) + + // Do a non-blocking signal to persist here. Our channel has a buffer of 1, + // so if we've signalled for some other Set call that hasn't been persisted + // yet, this Set will get persisted along with that one and any other + // pending in-memory changes. + select { + case v.setCh <- struct{}{}: + default: + } +} + +// ErrDuplicateWatch is returned when Watch is called on a synced Viper which +// has already started a watch. +var ErrDuplicateWatch = vterrors.New(vtrpc.Code_FAILED_PRECONDITION, "duplicate watch") + +// Watch starts watching the config used by the passed-in Viper. Before starting +// the watch, the synced viper will perform an initial read and load from disk +// so that the live config is ready for use without requiring an initial config +// change. +// +// If the given static viper did not load a config file (and is instead relying +// purely on defaults, flags, and environment variables), then the settings of +// that viper are merged over, and this synced Viper may be used to set up an +// actual watch later. Additionally, this starts a background goroutine to +// persist changes made in-memory back to disk. It returns a cancel func to stop +// the persist loop, which the caller is responsible for calling during +// shutdown (see package servenv for an example). +// +// This does two things — one which is a nice-to-have, and another which is +// necessary for correctness. +// +// 1. Writing in-memory changes (which usually occur through a request to a +// /debug/env endpoint) ensures they are persisted across process restarts. +// 2. Writing in-memory changes ensures that subsequent modifications to the +// config file do not clobber those changes. Because viper loads the entire +// config on-change, rather than an incremental (diff) load, if a user were to +// edit an unrelated key (keyA) in the file, and we did not persist the +// in-memory change (keyB), then future calls to keyB.Get() would return the +// older value. +// +// If this synced viper is already watching a config file, this function returns +// an ErrDuplicateWatch. Other errors may be returned via underlying viper code +// to ensure the config file can be read in properly. +func (v *Viper) Watch(ctx context.Context, static *viper.Viper, minWaitInterval time.Duration) (cancel context.CancelFunc, err error) { + if v.watchingConfig { + return nil, vterrors.Wrapf(ErrDuplicateWatch, "%s: viper is already watching %s", ErrDuplicateWatch.Error(), v.disk.ConfigFileUsed()) + } + + ctx, cancel = context.WithCancel(ctx) + + cfg := static.ConfigFileUsed() + if cfg == "" { + // No config file to watch, just merge the settings and return. + return cancel, v.live.MergeConfigMap(static.AllSettings()) + } + + v.disk.SetConfigFile(cfg) + if err := v.disk.ReadInConfig(); err != nil { + return nil, err + } + + v.watchingConfig = true + v.loadFromDisk() + v.disk.OnConfigChange(func(in fsnotify.Event) { + for _, m := range v.keys { + m.Lock() + // This won't fire until after the config has been updated on v.live. + defer m.Unlock() + } + + v.loadFromDisk() + + for _, ch := range v.subscribers { + select { + case ch <- struct{}{}: + default: + } + } + }) + v.disk.WatchConfig() + + go v.persistChanges(ctx, minWaitInterval) + + return cancel, nil +} + +func (v *Viper) persistChanges(ctx context.Context, minWaitInterval time.Duration) { + defer close(v.setCh) + + var timer *time.Timer + if minWaitInterval > 0 { + timer = time.NewTimer(minWaitInterval) + } + + persistOnce := func() { + if err := v.WriteConfig(); err != nil { + log.ERROR("failed to persist config changes back to disk: %s", err.Error()) + // If we failed to persist, don't wait the entire interval before + // writing again, instead writing immediately on the next request. + if timer != nil { + if !timer.Stop() { + <-timer.C + } + + timer = nil + } + } + + switch { + case minWaitInterval == 0: + return + case timer == nil: + timer = time.NewTimer(minWaitInterval) + default: + timer.Reset(minWaitInterval) + } + } + + for { + select { + case <-ctx.Done(): + return + case <-v.setCh: + if timer == nil { + persistOnce() + continue + } + + select { + case <-ctx.Done(): + return + case <-timer.C: + persistOnce() + } + } + } +} + +// WriteConfig writes the live viper config back to disk. +func (v *Viper) WriteConfig() error { + for _, m := range v.keys { + m.Lock() + // This won't fire until after the config has been written. + defer m.Unlock() + } + + v.m.Lock() + defer v.m.Unlock() + + v.live.SetConfigFile(v.disk.ConfigFileUsed()) + + return v.live.WriteConfig() +} + +// Notify adds a subscription that this synced viper will attempt to notify on +// config changes, after the updated config has been copied over from disk to +// live. +// +// Analogous to signal.Notify, notifications are sent non-blocking, so users +// should account for this when consuming from the channel they've provided. +// +// This function must be called prior to setting up a Watch; it will panic if a +// a watch has already been established on this synced Viper. +func (v *Viper) Notify(ch chan<- struct{}) { + if v.watchingConfig { + panic("cannot Notify after starting to watch a config") + } + + v.subscribers = append(v.subscribers, ch) +} + +// AllSettings returns the current live settings. +func (v *Viper) AllSettings() map[string]any { + v.m.Lock() + defer v.m.Unlock() + + return v.live.AllSettings() +} + +func (v *Viper) loadFromDisk() { + v.m.Lock() + defer v.m.Unlock() + + // Reset v.live so explicit Set calls don't win over what's just changed on + // disk. + v.live = viper.New() + + // Fun fact! MergeConfigMap actually only ever returns nil. Maybe in an + // older version of viper it used to actually handle errors, but now it + // decidedly does not. See https://github.com/spf13/viper/blob/v1.8.1/viper.go#L1492-L1499. + _ = v.live.MergeConfigMap(v.disk.AllSettings()) +} + +// begin implementation of registry.Bindable for sync.Viper + +func (v *Viper) BindEnv(vars ...string) error { return v.disk.BindEnv(vars...) } +func (v *Viper) BindPFlag(key string, flag *pflag.Flag) error { return v.disk.BindPFlag(key, flag) } +func (v *Viper) RegisterAlias(alias string, key string) { v.disk.RegisterAlias(alias, key) } +func (v *Viper) SetDefault(key string, value any) { v.disk.SetDefault(key, value) } + +// end implementation of registry.Bindable for sync.Viper + +// AdaptGetter wraps a get function (matching the signature of +// viperutil.Options.GetFunc) to be threadsafe with the passed-in synced Viper. +// +// It must be called prior to starting a watch on the synced Viper; it will +// panic if a watch has already been established. +// +// This function must be called at most once per key; it will panic if attempting +// to adapt multiple getters for the same key. +func AdaptGetter[T any](key string, getter func(v *viper.Viper) func(key string) T, v *Viper) func(key string) T { + if v.watchingConfig { + panic("cannot adapt getter to synchronized viper which is already watching a config") + } + + if _, ok := v.keys[key]; ok { + panic(fmt.Sprintf("already adapted a getter for key %s", key)) + } + + var m sync.RWMutex + v.keys[key] = &m + + return func(key string) T { + m.RLock() + defer m.RUnlock() + + return getter(v.live)(key) + } +} diff --git a/go/viperutil/internal/sync/sync_test.go b/go/viperutil/internal/sync/sync_test.go new file mode 100644 index 00000000000..fcafb941f48 --- /dev/null +++ b/go/viperutil/internal/sync/sync_test.go @@ -0,0 +1,258 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sync_test + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/viperutil" + vipersync "vitess.io/vitess/go/viperutil/internal/sync" + "vitess.io/vitess/go/viperutil/internal/value" +) + +func TestPersistConfig(t *testing.T) { + type config struct { + Foo int `json:"foo"` + } + + loadConfig := func(t *testing.T, f *os.File) config { + t.Helper() + + data, err := os.ReadFile(f.Name()) + require.NoError(t, err) + + var cfg config + require.NoError(t, json.Unmarshal(data, &cfg)) + + return cfg + } + + setup := func(t *testing.T, v *vipersync.Viper, minWaitInterval time.Duration) (*os.File, chan struct{}) { + tmp, err := os.CreateTemp(t.TempDir(), fmt.Sprintf("%s_*.json", strings.ReplaceAll(t.Name(), "/", "_"))) + require.NoError(t, err) + + t.Cleanup(func() { os.Remove(tmp.Name()) }) + + cfg := config{ + Foo: jitter(1, 100), + } + + data, err := json.Marshal(&cfg) + require.NoError(t, err) + + _, err = tmp.Write(data) + require.NoError(t, err) + + static := viper.New() + static.SetConfigFile(tmp.Name()) + require.NoError(t, static.ReadInConfig()) + + ch := make(chan struct{}, 1) + v.Notify(ch) + + cancel, err := v.Watch(context.Background(), static, minWaitInterval) + require.NoError(t, err) + t.Cleanup(cancel) + + return tmp, ch + } + + t.Run("basic", func(t *testing.T) { + v := vipersync.New() + + minPersistWaitInterval := 10 * time.Second + get := vipersync.AdaptGetter("foo", viperutil.GetFuncForType[int](), v) + f, ch := setup(t, v, minPersistWaitInterval) + + old := get("foo") + loadConfig(t, f) + v.Set("foo", old+1) + // This should happen immediately in-memory and on-disk. + assert.Equal(t, old+1, get("foo")) + <-ch + assert.Equal(t, old+1, loadConfig(t, f).Foo) + + v.Set("foo", old+2) + // This should _also_ happen immediately in-memory, but not on-disk. + // It will take up to 2 * minPersistWaitInterval to reach the disk. + assert.Equal(t, old+2, get("foo")) + assert.Equal(t, old+1, loadConfig(t, f).Foo) + + select { + case <-ch: + case <-time.After(2 * minPersistWaitInterval): + assert.Fail(t, "config was not persisted quickly enough", "config took longer than %s to persist (minPersistWaitInterval = %s)", 2*minPersistWaitInterval, minPersistWaitInterval) + } + + assert.Equal(t, old+2, loadConfig(t, f).Foo) + }) + + t.Run("no persist interval", func(t *testing.T) { + v := vipersync.New() + + var minPersistWaitInterval time.Duration + get := vipersync.AdaptGetter("foo", viperutil.GetFuncForType[int](), v) + f, ch := setup(t, v, minPersistWaitInterval) + + old := get("foo") + loadConfig(t, f) + v.Set("foo", old+1) + // This should happen immediately in-memory and on-disk. + assert.Equal(t, old+1, get("foo")) + <-ch + assert.Equal(t, old+1, loadConfig(t, f).Foo) + + v.Set("foo", old+2) + // This should _also_ happen immediately in-memory, and on-disk. + assert.Equal(t, old+2, get("foo")) + <-ch + assert.Equal(t, old+2, loadConfig(t, f).Foo) + }) +} + +func TestWatchConfig(t *testing.T) { + type config struct { + A, B int + } + + tmp, err := os.CreateTemp(".", "TestWatchConfig_*.json") + require.NoError(t, err) + t.Cleanup(func() { os.Remove(tmp.Name()) }) + + stat, err := os.Stat(tmp.Name()) + require.NoError(t, err) + + writeConfig := func(a, b int) error { + data, err := json.Marshal(&config{A: a, B: b}) + if err != nil { + return err + } + + return os.WriteFile(tmp.Name(), data, stat.Mode()) + } + writeRandomConfig := func() error { + a, b := rand.Intn(100), rand.Intn(100) + return writeConfig(a, b) + } + + require.NoError(t, writeRandomConfig()) + + v := viper.New() + v.SetConfigFile(tmp.Name()) + require.NoError(t, v.ReadInConfig()) + + wCh, rCh := make(chan struct{}), make(chan struct{}) + v.OnConfigChange(func(in fsnotify.Event) { + select { + case <-rCh: + return + default: + } + + wCh <- struct{}{} + // block forever to prevent this viper instance from double-updating. + <-rCh + }) + v.WatchConfig() + + // Make sure that basic, unsynchronized WatchConfig is set up before + // beginning the actual test. + a, b := v.GetInt("a"), v.GetInt("b") + require.NoError(t, writeConfig(a+1, b+1)) + <-wCh // wait for the update to finish + + require.Equal(t, a+1, v.GetInt("a")) + require.Equal(t, b+1, v.GetInt("b")) + + rCh <- struct{}{} + + sv := vipersync.New() + A := viperutil.Configure("a", viperutil.Options[int]{Dynamic: true}) + B := viperutil.Configure("b", viperutil.Options[int]{Dynamic: true}) + + A.(*value.Dynamic[int]).Base.BoundGetFunc = vipersync.AdaptGetter("a", func(v *viper.Viper) func(key string) int { + return v.GetInt + }, sv) + B.(*value.Dynamic[int]).Base.BoundGetFunc = vipersync.AdaptGetter("b", func(v *viper.Viper) func(key string) int { + return v.GetInt + }, sv) + + cancel, err := sv.Watch(context.Background(), v, 0) + require.NoError(t, err) + defer cancel() + + var wg sync.WaitGroup + ctx, cancel := context.WithCancel(context.Background()) + + // Sleep between 25 and 50ms between reads. + readJitter := func() time.Duration { + return time.Duration(jitter(25, 50)) * time.Millisecond + } + // Sleep between 75 and 125ms between writes. + writeJitter := func() time.Duration { + return time.Duration(jitter(75, 125)) * time.Millisecond + } + + for i := 0; i < 10; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + + for { + select { + case <-ctx.Done(): + return + default: + } + + switch i % 2 { + case 0: + A.Get() + case 1: + B.Get() + } + + time.Sleep(readJitter()) + } + }(i) + } + + for i := 0; i < 100; i++ { + require.NoError(t, writeRandomConfig()) + time.Sleep(writeJitter()) + } + + cancel() + wg.Wait() +} + +func jitter(min, max int) int { + return min + rand.Intn(max-min+1) +} diff --git a/go/viperutil/internal/value/value.go b/go/viperutil/internal/value/value.go new file mode 100644 index 00000000000..a958d642bcf --- /dev/null +++ b/go/viperutil/internal/value/value.go @@ -0,0 +1,170 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package value + +import ( + "fmt" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/internal/registry" + "vitess.io/vitess/go/viperutil/internal/sync" + "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/vterrors" +) + +// Registerable is the subset of the interface exposed by Values (which is +// declared in the public viperutil package). +// +// We need a separate interface type because Go generics do not let you define +// a function that takes Value[T] for many, different T's, which we want to do +// for BindFlags. +type Registerable interface { + Key() string + Registry() registry.Bindable + Flag(fs *pflag.FlagSet) (*pflag.Flag, error) +} + +// Base is the base functionality shared by Static and Dynamic values. It +// implements viperutil.Value. +type Base[T any] struct { + KeyName string + DefaultVal T + + GetFunc func(v *viper.Viper) func(key string) T + BoundGetFunc func(key string) T + + Aliases []string + FlagName string + EnvVars []string +} + +func (val *Base[T]) Key() string { return val.KeyName } +func (val *Base[T]) Default() T { return val.DefaultVal } +func (val *Base[T]) Get() T { return val.BoundGetFunc(val.Key()) } + +// ErrNoFlagDefined is returned when a Value has a FlagName set, but the given +// FlagSet does not define a flag with that name. +var ErrNoFlagDefined = vterrors.New(vtrpc.Code_INVALID_ARGUMENT, "flag not defined") + +// Flag is part of the Registerable interface. If the given flag set has a flag +// with the name of this value's configured flag, that flag is returned, along +// with a nil error. If no flag exists on the flag set with that name, an error +// is returned. +// +// If the value is not configured to correspond to a flag (FlagName == ""), then +// (nil, nil) is returned. +func (val *Base[T]) Flag(fs *pflag.FlagSet) (*pflag.Flag, error) { + if val.FlagName == "" { + return nil, nil + } + + flag := fs.Lookup(val.FlagName) + if flag == nil { + return nil, vterrors.Wrapf(ErrNoFlagDefined, "%s with name %s (for key %s)", ErrNoFlagDefined.Error(), val.FlagName, val.Key()) + } + + return flag, nil +} + +func (val *Base[T]) bind(v registry.Bindable) { + v.SetDefault(val.Key(), val.DefaultVal) + + for _, alias := range val.Aliases { + v.RegisterAlias(alias, val.Key()) + } + + if len(val.EnvVars) > 0 { + vars := append([]string{val.Key()}, val.EnvVars...) + _ = v.BindEnv(vars...) + } +} + +// BindFlags creates bindings between each value's registry and the given flag +// set. This function will panic if any of the values defines a flag that does +// not exist in the flag set. +func BindFlags(fs *pflag.FlagSet, values ...Registerable) { + for _, val := range values { + flag, err := val.Flag(fs) + switch { + case err != nil: + panic(fmt.Errorf("failed to load flag for %s: %w", val.Key(), err)) + case flag == nil: + continue + } + + _ = val.Registry().BindPFlag(val.Key(), flag) + if flag.Name != val.Key() { + val.Registry().RegisterAlias(flag.Name, val.Key()) + } + } +} + +// Static is a static value. Static values register to the Static registry, and +// do not respond to changes to config files. Their Get() method will return the +// same value for the lifetime of the process. +type Static[T any] struct { + *Base[T] +} + +// NewStatic returns a static value derived from the given base value, after +// binding it to the static registry. +func NewStatic[T any](base *Base[T]) *Static[T] { + base.bind(registry.Static) + base.BoundGetFunc = base.GetFunc(registry.Static) + + return &Static[T]{ + Base: base, + } +} + +func (val *Static[T]) Registry() registry.Bindable { + return registry.Static +} + +func (val *Static[T]) Set(v T) { + registry.Static.Set(val.KeyName, v) +} + +// Dynamic is a dynamic value. Dynamic values register to the Dynamic registry, +// and respond to changes to watched config files. Their Get() methods will +// return whatever value is currently live in the config, in a threadsafe +// manner. +type Dynamic[T any] struct { + *Base[T] +} + +// NewDynamic returns a dynamic value derived from the given base value, after +// binding it to the dynamic registry and wrapping its GetFunc to be threadsafe +// with respect to config reloading. +func NewDynamic[T any](base *Base[T]) *Dynamic[T] { + base.bind(registry.Dynamic) + base.BoundGetFunc = sync.AdaptGetter(base.Key(), base.GetFunc, registry.Dynamic) + + return &Dynamic[T]{ + Base: base, + } +} + +func (val *Dynamic[T]) Registry() registry.Bindable { + return registry.Dynamic +} + +func (val *Dynamic[T]) Set(v T) { + registry.Dynamic.Set(val.KeyName, v) +} diff --git a/go/viperutil/value.go b/go/viperutil/value.go new file mode 100644 index 00000000000..4433b53b05d --- /dev/null +++ b/go/viperutil/value.go @@ -0,0 +1,63 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package viperutil + +import ( + "github.com/spf13/pflag" + + "vitess.io/vitess/go/viperutil/internal/value" +) + +var ( + _ Value[int] = (*value.Static[int])(nil) + _ Value[int] = (*value.Dynamic[int])(nil) +) + +// Value represents the public API to access viper-backed config values. +// +// N.B. the embedded value.Registerable interface is necessary only for +// BindFlags and other mechanisms of binding Values to the internal registries +// to work. Users of Value objects should only need to call Get(), Set(v T), and +// Default(). +type Value[T any] interface { + value.Registerable + + // Get returns the current value. For static implementations, this will + // never change after the initial config load. For dynamic implementations, + // this may change throughout the lifetime of the vitess process. + Get() T + // Set sets the underlying value. For both static and dynamic + // implementations, this is reflected in subsequent calls to Get. + // + // If a config file was loaded, changes to dynamic values will be persisted + // back to the config file in the background, governed by the behavior of + // the --config-persistence-min-interval flag. + Set(v T) + // Default returns the default value configured for this Value. For both + // static and dynamic implementations, it should never change. + Default() T +} + +// BindFlags binds a set of Registerable values to the given flag set. +// +// This function will panic if any of the values was configured to map to a flag +// which is not defined on the flag set. Therefore, this function should usually +// be called in an OnParse or OnParseFor hook after defining the flags for the +// values in question. +func BindFlags(fs *pflag.FlagSet, values ...value.Registerable) { + value.BindFlags(fs, values...) +} diff --git a/go/viperutil/viper.go b/go/viperutil/viper.go new file mode 100644 index 00000000000..7d120059587 --- /dev/null +++ b/go/viperutil/viper.go @@ -0,0 +1,189 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Package viperutil provides a utility layer to streamline and standardize +interacting with viper-backed configuration values across vitess components. + +The common pattern is for a given module to declare their values by declaring +variables that are the result of calling Configure, for example in package trace: + + package trace + + import "vitess.io/vitess/go/viperutil" + + var ( + modulePrefix = viperutil.KeyPrefixFunc("trace") + + tracingServer = viperutil.Configure( + modulePrefix("service"), + viperutil.Options[string]{ + Default: "noop", + FlagName: "tracer", + } + ) + enableLogging = viperutil.Configure( + modulePrefix("enable-logging"), + viperutil.Options[bool]{ + FlagName: "tracing-enable-logging", + } + ) + ) + +Then, in an OnParseFor or OnParse hook, declare any flags, and bind viper values +to those flags, as appropriate: + + package trace + + import ( + "github.com/spf13/pflag" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/vt/servenv" + ) + + func init() { + servenv.OnParse(func(fs *pflag.FlagSet) { + fs.String("tracer", tracingServer.Default(), "") + fs.Bool("tracing-enable-logging", enableLogging.Default(), "") + + viperutil.BindFlags(fs, tracingServer, enableLogging) + }) + } + +Finally, after a call to `viperutil.LoadConfig` (which is done as a part of +`servenv.ParseFlags`), values may be accessed by calling their `.Get()` methods. + +For more details, refer to the package documentation, as well as the documents +in doc/viper/. +*/ +package viperutil + +import ( + "strings" + + "github.com/spf13/viper" + + "vitess.io/vitess/go/viperutil/internal/value" +) + +// Options represents the various options used to control how Values are +// configured by viperutil. +type Options[T any] struct { + // Aliases, if set, configures the Value to be accessible via additional + // keys. + // + // This is useful for deprecating old names gracefully while maintaining + // backwards-compatibility. + Aliases []string + // FlagName, if set, allows a value to be configured to also check the + // named flag for its final config value. If depending on a flag, BindFlags + // must be called on the Value returned from Configure. In most cases, + // modules will do this in the same OnParse hook that defines their flags. + // + // Note that if the set FlagName does not match the Value's key, Configure + // will automatically register an alias to allow both names to be used as + // the key, which is necessary for the flag value to be discoverable by + // viper for the Value's actual key. + FlagName string + // EnvVars, if set, configures the Value to also check the given environment + // variables for its final config value. + // + // Note that unlike keys and aliases, environment variable names are + // case-sensitive. + EnvVars []string + // Default is the default value that will be set for the key. If not + // explicitly set during a call to Configure, the default value will be the + // zero value for the type T. This means if T is a pointer type, the default + // will be nil, not the zeroed out struct. + Default T + + // Dynamic, if set, configures a value to be backed by the dynamic registry. + // If a config file is used (via LoadConfig), that file will be watched for + // changes, and dynamic Values will reflect changes via their Get() methods + // (whereas static values will only ever return the value loaded initially). + Dynamic bool + + // GetFunc is the function used to get this value out of a viper. + // + // If omitted, GetFuncForType will attempt to provide a useful default for + // the given type T. For primitive types, this should be sufficient. For + // more fine-grained control over value retrieval and unmarshalling, callers + // should provide their own function. + // + // See GetFuncForType for further details. + GetFunc func(v *viper.Viper) func(key string) T +} + +// Configure configures a viper-backed value associated with the given key to +// either the static or dynamic internal registries, returning the resulting +// Value. This value is partially ready for use (it will be able to get values +// from environment variables and defaults), but file-based configs will not +// available until servenv calls LoadConfig, and flag-based configs will not be +// available until a combination of BindFlags and pflag.Parse have been called, +// usually by servenv. +// +// Exact behavior of how the key is bound to the registries depend on the +// Options provided, +func Configure[T any](key string, opts Options[T]) (v Value[T]) { + getfunc := opts.GetFunc + if getfunc == nil { + getfunc = GetFuncForType[T]() + } + + base := &value.Base[T]{ + KeyName: key, + DefaultVal: opts.Default, + GetFunc: getfunc, + Aliases: opts.Aliases, + FlagName: opts.FlagName, + EnvVars: opts.EnvVars, + } + + switch { + case opts.Dynamic: + v = value.NewDynamic(base) + default: + v = value.NewStatic(base) + } + + return v +} + +// KeyPrefixFunc is a helper function to allow modules to extract a common key +// prefix used by that module to avoid repitition (and typos, missed updates, +// and so on). +// +// For example, package go/vt/vttablet/schema may want to do: +// +// moduleKey := viperutil.KeyPrefixFunc("vttablet.schema") +// watch := viperutil.Configure(moduleKey("watch_interval"), ...) // => "vttablet.schema.watch_interval" +// // ... and so on +func KeyPrefixFunc(prefix string) func(subkey string) (fullkey string) { + var keyParts []string + if prefix != "" { + keyParts = append(keyParts, prefix) + } + + return func(subkey string) (fullkey string) { + tmp := keyParts + if subkey != "" { + tmp = append(tmp, subkey) + } + + return strings.Join(tmp, ".") + } +} diff --git a/go/viperutil/vipertest/stub.go b/go/viperutil/vipertest/stub.go new file mode 100644 index 00000000000..5c8d3e78b43 --- /dev/null +++ b/go/viperutil/vipertest/stub.go @@ -0,0 +1,60 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vipertest + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/viperutil" + "vitess.io/vitess/go/viperutil/internal/value" +) + +// Stub stubs out a given value to use the passed-in viper to retrieve its +// config value for testing purposes. It returns a function to undo this, +// resetting the Value to whatever registry (Static, or Dynamic) it was +// originally bound to. +// +// It fails the test if a caller attempts to stub the same value multiple times +// to a particular viper. +func Stub[T any](t *testing.T, v *viper.Viper, val viperutil.Value[T]) func() { + t.Helper() + + if !assert.False(t, v.InConfig(val.Key()), "value for key %s already stubbed", val.Key()) { + return func() {} + } + + var base *value.Base[T] + switch val := val.(type) { + case *value.Static[T]: + base = val.Base + case *value.Dynamic[T]: + base = val.Base + default: + assert.Fail(t, "value %+v does not support stubbing", val) + return func() {} + } + + oldGet := base.BoundGetFunc + base.BoundGetFunc = base.GetFunc(v) + + return func() { + base.BoundGetFunc = oldGet + } +} diff --git a/go/vt/discovery/replicationlag.go b/go/vt/discovery/replicationlag.go index 71ab78fd15b..8ae168ddff9 100644 --- a/go/vt/discovery/replicationlag.go +++ b/go/vt/discovery/replicationlag.go @@ -23,15 +23,44 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/servenv" ) var ( + configKey = viperutil.KeyPrefixFunc("discovery") // lowReplicationLag defines the duration that replication lag is low enough that the VTTablet is considered healthy. - lowReplicationLag time.Duration - highReplicationLagMinServing time.Duration - minNumTablets int - legacyReplicationLagAlgorithm bool + lowReplicationLag = viperutil.Configure( + configKey("low_replication_lag"), + viperutil.Options[time.Duration]{ + FlagName: "discovery_low_replication_lag", + Default: 30 * time.Second, + Dynamic: true, + }, + ) + highReplicationLagMinServing = viperutil.Configure( + configKey("high_replication_lag"), + viperutil.Options[time.Duration]{ + FlagName: "discovery_high_replication_lag_minimum_serving", + Default: 2 * time.Hour, + Dynamic: true, + }, + ) + minNumTablets = viperutil.Configure( + configKey("min_number_serving_vttablets"), + viperutil.Options[int]{ + FlagName: "min_number_serving_vttablets", + Default: 2, + Dynamic: true, + }, + ) + legacyReplicationLagAlgorithm = viperutil.Configure( + configKey("legacy_replication_lag_algorithm"), + viperutil.Options[bool]{ + FlagName: "legacy_replication_lag_algorithm", + Default: true, + }, + ) ) func init() { @@ -39,52 +68,59 @@ func init() { } func registerReplicationFlags(fs *pflag.FlagSet) { - fs.DurationVar(&lowReplicationLag, "discovery_low_replication_lag", 30*time.Second, "Threshold below which replication lag is considered low enough to be healthy.") - fs.DurationVar(&highReplicationLagMinServing, "discovery_high_replication_lag_minimum_serving", 2*time.Hour, "Threshold above which replication lag is considered too high when applying the min_number_serving_vttablets flag.") - fs.IntVar(&minNumTablets, "min_number_serving_vttablets", 2, "The minimum number of vttablets for each replicating tablet_type (e.g. replica, rdonly) that will be continue to be used even with replication lag above discovery_low_replication_lag, but still below discovery_high_replication_lag_minimum_serving.") - fs.BoolVar(&legacyReplicationLagAlgorithm, "legacy_replication_lag_algorithm", true, "Use the legacy algorithm when selecting vttablets for serving.") + fs.Duration("discovery_low_replication_lag", lowReplicationLag.Default(), "Threshold below which replication lag is considered low enough to be healthy.") + fs.Duration("discovery_high_replication_lag_minimum_serving", highReplicationLagMinServing.Default(), "Threshold above which replication lag is considered too high when applying the min_number_serving_vttablets flag.") + fs.Int("min_number_serving_vttablets", minNumTablets.Default(), "The minimum number of vttablets for each replicating tablet_type (e.g. replica, rdonly) that will be continue to be used even with replication lag above discovery_low_replication_lag, but still below discovery_high_replication_lag_minimum_serving.") + fs.Bool("legacy_replication_lag_algorithm", legacyReplicationLagAlgorithm.Default(), "Use the legacy algorithm when selecting vttablets for serving.") + + viperutil.BindFlags(fs, + lowReplicationLag, + highReplicationLagMinServing, + minNumTablets, + legacyReplicationLagAlgorithm, + ) } // GetLowReplicationLag getter for use by debugenv func GetLowReplicationLag() time.Duration { - return lowReplicationLag + return lowReplicationLag.Get() } // SetLowReplicationLag setter for use by debugenv func SetLowReplicationLag(lag time.Duration) { - lowReplicationLag = lag + lowReplicationLag.Set(lag) } // GetHighReplicationLagMinServing getter for use by debugenv func GetHighReplicationLagMinServing() time.Duration { - return highReplicationLagMinServing + return highReplicationLagMinServing.Get() } // SetHighReplicationLagMinServing setter for use by debugenv func SetHighReplicationLagMinServing(lag time.Duration) { - highReplicationLagMinServing = lag + highReplicationLagMinServing.Set(lag) } // GetMinNumTablets getter for use by debugenv func GetMinNumTablets() int { - return minNumTablets + return minNumTablets.Get() } // SetMinNumTablets setter for use by debugenv func SetMinNumTablets(numTablets int) { - minNumTablets = numTablets + minNumTablets.Set(numTablets) } // IsReplicationLagHigh verifies that the given LegacytabletHealth refers to a tablet with high // replication lag, i.e. higher than the configured discovery_low_replication_lag flag. func IsReplicationLagHigh(tabletHealth *TabletHealth) bool { - return float64(tabletHealth.Stats.ReplicationLagSeconds) > lowReplicationLag.Seconds() + return float64(tabletHealth.Stats.ReplicationLagSeconds) > lowReplicationLag.Get().Seconds() } // IsReplicationLagVeryHigh verifies that the given LegacytabletHealth refers to a tablet with very high // replication lag, i.e. higher than the configured discovery_high_replication_lag_minimum_serving flag. func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { - return float64(tabletHealth.Stats.ReplicationLagSeconds) > highReplicationLagMinServing.Seconds() + return float64(tabletHealth.Stats.ReplicationLagSeconds) > highReplicationLagMinServing.Get().Seconds() } // FilterStatsByReplicationLag filters the list of TabletHealth by TabletHealth.Stats.ReplicationLagSeconds. @@ -113,13 +149,13 @@ func IsReplicationLagVeryHigh(tabletHealth *TabletHealth) bool { // - degraded_threshold: this is only used by vttablet for display. It should match // discovery_low_replication_lag here, so the vttablet status display matches what vtgate will do of it. func FilterStatsByReplicationLag(tabletHealthList []*TabletHealth) []*TabletHealth { - if !legacyReplicationLagAlgorithm { + if !legacyReplicationLagAlgorithm.Get() { return filterStatsByLag(tabletHealthList) } res := filterStatsByLagWithLegacyAlgorithm(tabletHealthList) // run the filter again if exactly one tablet is removed, // and we have spare tablets. - if len(res) > minNumTablets && len(res) == len(tabletHealthList)-1 { + if len(res) > minNumTablets.Get() && len(res) == len(tabletHealthList)-1 { res = filterStatsByLagWithLegacyAlgorithm(res) } return res @@ -145,7 +181,7 @@ func filterStatsByLag(tabletHealthList []*TabletHealth) []*TabletHealth { // Pick those with low replication lag, but at least minNumTablets tablets regardless. res := make([]*TabletHealth, 0, len(list)) for i := 0; i < len(list); i++ { - if !IsReplicationLagHigh(list[i].ts) || i < minNumTablets { + if !IsReplicationLagHigh(list[i].ts) || i < minNumTablets.Get() { res = append(res, list[i].ts) } } @@ -186,7 +222,7 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*Ta res = append(res, ts) } } - if len(res) >= minNumTablets { + if len(res) >= minNumTablets.Get() { return res } // return at least minNumTablets tablets to avoid over loading, @@ -219,8 +255,8 @@ func filterStatsByLagWithLegacyAlgorithm(tabletHealthList []*TabletHealth) []*Ta sort.Sort(byReplag(snapshots)) // Pick the first minNumTablets tablets. - res = make([]*TabletHealth, 0, minNumTablets) - for i := 0; i < min(minNumTablets, len(snapshots)); i++ { + res = make([]*TabletHealth, 0, minNumTablets.Get()) + for i := 0; i < min(minNumTablets.Get(), len(snapshots)); i++ { res = append(res, snapshots[i].ts) } return res diff --git a/go/vt/discovery/replicationlag_test.go b/go/vt/discovery/replicationlag_test.go index 80c3921d052..9c047469fb2 100644 --- a/go/vt/discovery/replicationlag_test.go +++ b/go/vt/discovery/replicationlag_test.go @@ -28,20 +28,20 @@ import ( ) func init() { - lowReplicationLag = 30 * time.Second - highReplicationLagMinServing = 2 * time.Hour - minNumTablets = 2 - legacyReplicationLagAlgorithm = true + lowReplicationLag.Set(30 * time.Second) + highReplicationLagMinServing.Set(2 * time.Hour) + minNumTablets.Set(2) + legacyReplicationLagAlgorithm.Set(true) } // testSetLegacyReplicationLagAlgorithm is a test helper function, if this is used by a production code path, something is wrong. func testSetLegacyReplicationLagAlgorithm(newLegacy bool) { - legacyReplicationLagAlgorithm = newLegacy + legacyReplicationLagAlgorithm.Set(newLegacy) } // testSetMinNumTablets is a test helper function, if this is used by a production code path, something is wrong. func testSetMinNumTablets(newMin int) { - minNumTablets = newMin + minNumTablets.Set(newMin) } func TestFilterByReplicationLagUnhealthy(t *testing.T) { diff --git a/go/vt/discovery/tablet_health_check.go b/go/vt/discovery/tablet_health_check.go index b3cc06e2d10..95821db88a2 100644 --- a/go/vt/discovery/tablet_health_check.go +++ b/go/vt/discovery/tablet_health_check.go @@ -212,7 +212,7 @@ func (thc *tabletHealthCheck) isTrivialReplagChange(newStats *query.RealtimeStat } // Skip replag filter when replag remains in the low rep lag range, // which should be the case majority of the time. - lowRepLag := lowReplicationLag.Seconds() + lowRepLag := lowReplicationLag.Get().Seconds() oldRepLag := float64(thc.Stats.ReplicationLagSeconds) newRepLag := float64(newStats.ReplicationLagSeconds) if oldRepLag <= lowRepLag && newRepLag <= lowRepLag { diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index 08fa24643b7..2f382bdbfc6 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -32,6 +32,7 @@ import ( "github.com/Azure/azure-storage-blob-go/azblob" "github.com/spf13/pflag" + "vitess.io/vitess/go/viperutil" "vitess.io/vitess/go/vt/concurrency" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" @@ -40,26 +41,59 @@ import ( var ( // This is the account name - accountName string + accountName = viperutil.Configure( + configKey("account.name"), + viperutil.Options[string]{ + EnvVars: []string{"VT_AZBLOB_ACCOUNT_NAME"}, + FlagName: "azblob_backup_account_name", + }, + ) // This is the private access key - accountKeyFile string + accountKeyFile = viperutil.Configure( + configKey("account.key_file"), + viperutil.Options[string]{ + FlagName: "azblob_backup_account_key_file", + }, + ) // This is the name of the container that will store the backups - containerName string + containerName = viperutil.Configure( + configKey("container_name"), + viperutil.Options[string]{ + FlagName: "azblob_backup_container_name", + }, + ) // This is an optional prefix to prepend to all files - storageRoot string + storageRoot = viperutil.Configure( + configKey("storage_root"), + viperutil.Options[string]{ + FlagName: "azblob_backup_storage_root", + }, + ) - azBlobParallelism int + azBlobParallelism = viperutil.Configure( + configKey("parallelism"), + viperutil.Options[int]{ + Default: 1, + FlagName: "azblob_backup_parallelism", + }, + ) ) +const configKeyPrefix = "backup.storage.azblob" + +var configKey = viperutil.KeyPrefixFunc(configKeyPrefix) + func registerFlags(fs *pflag.FlagSet) { - fs.StringVar(&accountName, "azblob_backup_account_name", "", "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") - fs.StringVar(&accountKeyFile, "azblob_backup_account_key_file", "", "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") - fs.StringVar(&containerName, "azblob_backup_container_name", "", "Azure Blob Container Name.") - fs.StringVar(&storageRoot, "azblob_backup_storage_root", "", "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") - fs.IntVar(&azBlobParallelism, "azblob_backup_parallelism", 1, "Azure Blob operation parallelism (requires extra memory when increased).") + fs.String("azblob_backup_account_name", accountName.Default(), "Azure Storage Account name for backups; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_NAME will be used.") + fs.String("azblob_backup_account_key_file", accountKeyFile.Default(), "Path to a file containing the Azure Storage account key; if this flag is unset, the environment variable VT_AZBLOB_ACCOUNT_KEY will be used as the key itself (NOT a file path).") + fs.String("azblob_backup_container_name", containerName.Default(), "Azure Blob Container Name.") + fs.String("azblob_backup_storage_root", storageRoot.Default(), "Root prefix for all backup-related Azure Blobs; this should exclude both initial and trailing '/' (e.g. just 'a/b' not '/a/b/').") + fs.Int("azblob_backup_parallelism", azBlobParallelism.Default(), "Azure Blob operation parallelism (requires extra memory when increased).") + + viperutil.BindFlags(fs, accountName, accountKeyFile, containerName, storageRoot, azBlobParallelism) } func init() { @@ -79,16 +113,12 @@ const ( // 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key) // 2. Environment variables func azInternalCredentials() (string, string, error) { - actName := accountName - if actName == "" { - // Check the Environmental Value - actName = os.Getenv("VT_AZBLOB_ACCOUNT_NAME") - } + actName := accountName.Get() var actKey string - if accountKeyFile != "" { - log.Infof("Getting Azure Storage Account key from file: %s", accountKeyFile) - dat, err := os.ReadFile(accountKeyFile) + if keyFile := accountKeyFile.Get(); keyFile != "" { + log.Infof("Getting Azure Storage Account key from file: %s", keyFile) + dat, err := os.ReadFile(keyFile) if err != nil { return "", "", err } @@ -219,7 +249,7 @@ func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, file defer bh.waitGroup.Done() _, err := azblob.UploadStreamToBlockBlob(bh.ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{ BufferSize: azblob.BlockBlobMaxStageBlockBytes, - MaxBuffers: azBlobParallelism, + MaxBuffers: azBlobParallelism.Get(), }) if err != nil { reader.CloseWithError(err) @@ -286,7 +316,7 @@ func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) { if err != nil { return nil, err } - u := azServiceURL(credentials).NewContainerURL(containerName) + u := azServiceURL(credentials).NewContainerURL(containerName.Get()) return &u, nil } @@ -425,8 +455,8 @@ func (bs *AZBlobBackupStorage) WithParams(params backupstorage.Params) backupsto // Unlike path.Join, it doesn't collapse ".." or strip trailing slashes. // It also adds the value of the -azblob_backup_storage_root flag if set. func objName(parts ...string) string { - if storageRoot != "" { - return storageRoot + "/" + strings.Join(parts, "/") + if root := storageRoot.Get(); root != "" { + return root + "/" + strings.Join(parts, "/") } return strings.Join(parts, "/") } diff --git a/go/vt/servenv/servenv.go b/go/vt/servenv/servenv.go index e4464020cbf..662e4da5207 100644 --- a/go/vt/servenv/servenv.go +++ b/go/vt/servenv/servenv.go @@ -44,14 +44,13 @@ import ( "vitess.io/vitess/go/netutil" "vitess.io/vitess/go/stats" "vitess.io/vitess/go/trace" + "vitess.io/vitess/go/viperutil" + viperdebug "vitess.io/vitess/go/viperutil/debug" "vitess.io/vitess/go/vt/grpccommon" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/vterrors" - // register the proper init and shutdown hooks for logging - _ "vitess.io/vitess/go/vt/logutil" - // Include deprecation warnings for soon-to-be-unsupported flag invocations. _flag "vitess.io/vitess/go/internal/flag" ) @@ -323,11 +322,17 @@ func getFlagHooksFor(cmd string) (hooks []func(fs *pflag.FlagSet)) { return hooks } +// Needed because some tests require multiple parse passes, so we guard against +// that here. +var debugConfigRegisterOnce sync.Once + // ParseFlags initializes flags and handles the common case when no positional // arguments are expected. func ParseFlags(cmd string) { fs := GetFlagSetFor(cmd) + viperutil.BindFlags(fs) + _flag.Parse(fs) if version { @@ -341,6 +346,8 @@ func ParseFlags(cmd string) { log.Exitf("%s doesn't take any positional arguments, got '%s'", cmd, strings.Join(args, " ")) } + loadViper(cmd) + logutil.PurgeLogs() } @@ -359,6 +366,8 @@ func GetFlagSetFor(cmd string) *pflag.FlagSet { func ParseFlagsWithArgs(cmd string) []string { fs := GetFlagSetFor(cmd) + viperutil.BindFlags(fs) + _flag.Parse(fs) if version { @@ -371,11 +380,24 @@ func ParseFlagsWithArgs(cmd string) []string { log.Exitf("%s expected at least one positional argument", cmd) } + loadViper(cmd) + logutil.PurgeLogs() return args } +func loadViper(cmd string) { + watchCancel, err := viperutil.LoadConfig() + if err != nil { + log.Exitf("%s: failed to read in config: %s", cmd, err.Error()) + } + OnTerm(watchCancel) + debugConfigRegisterOnce.Do(func() { + HTTPHandleFunc("/debug/config", viperdebug.HandlerFunc) + }) +} + // Flag installations for packages that servenv imports. We need to register // here rather than in those packages (which is what we would normally do) // because that would create a dependency cycle. @@ -428,6 +450,8 @@ func init() { OnParse(log.RegisterFlags) // Flags in package logutil are installed for all binaries. OnParse(logutil.RegisterFlags) + // Flags in package viperutil/config are installed for all binaries. + OnParse(viperutil.RegisterFlags) } func RegisterFlagsForTopoBinaries(registerFlags func(fs *pflag.FlagSet)) {