From de1b04d85a587cbea5ee5d0d221642ec7421cf83 Mon Sep 17 00:00:00 2001 From: Robert Butts Date: Mon, 22 Aug 2022 16:21:42 -0600 Subject: [PATCH] Add t3c flag for local ATS version for config gen --- CHANGELOG.md | 1 + cache-config/t3c-apply/README.md | 202 ++++++++------- cache-config/t3c-apply/config/config.go | 32 +++ cache-config/t3c-apply/torequest/cmd.go | 3 + cache-config/t3c-generate/README.md | 7 + cache-config/t3c-generate/cfgfile/all.go | 2 +- cache-config/t3c-generate/cfgfile/wrappers.go | 33 ++- cache-config/t3c-generate/config/config.go | 14 + lib/go-atscfg/atscfg.go | 36 ++- lib/go-atscfg/atscfg_test.go | 6 +- lib/go-atscfg/headerrewritedotconfig.go | 15 +- lib/go-atscfg/loggingdotyaml.go | 22 +- lib/go-atscfg/meta.go | 20 +- lib/go-atscfg/parentabstraction.go | 6 +- lib/go-atscfg/parentdotconfig.go | 67 ++--- lib/go-atscfg/parentdotconfig_test.go | 245 ++++++++++++++++++ lib/go-atscfg/remapdotconfig.go | 20 +- lib/go-atscfg/strategiesdotconfig.go | 28 +- 18 files changed, 579 insertions(+), 180 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e413a436bc..2bed4498ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Added - [#6033](https://github.com/apache/trafficcontrol/issues/6033) [Traffic Ops, Traffic Portal] Added ability to assign multiple server capabilities to a server. +- [#7032](https://github.com/apache/trafficcontrol/issues/7032) Add t3c-apply flag to use local ATS version for config generation rather than Server package Parameter, to allow managing the ATS OS package via external tools. See 'man t3c-apply' and 'man t3c-generate' for details. - [Traffic Monitor] Added logging for `ipv4Availability` and `ipv6Availability` in TM. ### Fixed diff --git a/cache-config/t3c-apply/README.md b/cache-config/t3c-apply/README.md index deddfd64fd..8f16e760db 100644 --- a/cache-config/t3c-apply/README.md +++ b/cache-config/t3c-apply/README.md @@ -54,189 +54,207 @@ Typical usage is to install t3c on the cache machine, and then run it periodical -2, -\-default-client-enable-h2 - Whether to enable HTTP/2 on Delivery Services by default, if - they have no explicit Parameter. This is irrelevant if ATS - records.config is not serving H2. If omitted, H2 is - disabled. + Whether to enable HTTP/2 on Delivery Services by default, if + they have no explicit Parameter. This is irrelevant if ATS + records.config is not serving H2. If omitted, H2 is + disabled. -a, -\-service-action=value - action to perform on Traffic Server and other system - services. Only reloads if necessary, but always restarts. - Default is 'reload' + The action to perform on Traffic Server and other system + services. Only reloads if necessary, but always restarts. + Default is 'reload' -A, -\-update-ipallow - Whether ipallow file will be updated if necessary. This - exists because ATS had a bug where reloading after changing - ipallow would block everything. Default is false. + Whether ipallow file will be updated if necessary. This + exists because ATS had a bug where reloading after changing + ipallow would block everything. Default is false. + -b, -\-dns-local-bind - [true | false] whether to use the server's Service Addresses - to set the ATS DNS local bind address + [true | false] whether to use the server's Service Addresses + to set the ATS DNS local bind address. -c, -\-disable-parent-config-comments - Whether to disable verbose parent.config comments. Default - false. + Whether to disable verbose parent.config comments. Default + false. -C, -\-skip-os-check - [false | true] skip os check, default is false + [false | true] skip os check, default is false -d, -\-no-unset-update-flag - Whether to not unset the update flag in Traffic Ops after - applying files. This option makes it possible to generate - test or debug configuration from a production Traffic Ops - without un-setting queue or reval flags. Default is false. + Whether to not unset the update flag in Traffic Ops after + applying files. This option makes it possible to generate + test or debug configuration from a production Traffic Ops + without un-setting queue or reval flags. Default is false. -e, -\-omit-via-string-release - Whether to set the records.config via header to the ATS - release from the RPM. Default true. + Whether to set the records.config via header to the ATS + release from the RPM. Default true. -E, -\-version - Print version information and exit. + Print version information and exit. -f, -\-files=value [all | reval] - Which files to generate. If reval, the Traffic - Ops server reval_pending flag is used instead of the - upd_pending flag. Default is 'all' + Which files to generate. If reval, the Traffic + Ops server reval_pending flag is used instead of the + upd_pending flag. Default is 'all'. -F, -\-ignore-update-flag - Whether to ignore the upd_pending or reval_pending flag in - Traffic Ops, and always generate and apply files. If true, - the flag is still unset in Traffic Ops after files are - applied. Default is false. + Whether to ignore the upd_pending or reval_pending flag in + Traffic Ops, and always generate and apply files. If true, + the flag is still unset in Traffic Ops after files are + applied. Default is false. -g, -\-git=value - Create and use a git repo in the config directory. Options - are yes, no, and auto. If yes, create and use. If auto, use - if it exist. Default is auto. [auto] + + Create and use a git repo in the config directory. Options + are yes, no, and auto. If yes, create and use. If auto, use + if it exist. Default is auto. [auto] -H, -\-cache-host-name=value - Host name of the cache to generate config for. Must be the - server host name in Traffic Ops, not a URL, and not the FQDN + Host name of the cache to generate config for. Must be the + server host name in Traffic Ops, not a URL, and not the FQDN -h, -\-help - Print usage information and exit + Print usage information and exit -i, -\-no-outgoing-ip - Whether to not set the records.config outgoing IP to the - server's addresses in Traffic Ops. Default is false. + Whether to not set the records.config outgoing IP to the + server's addresses in Traffic Ops. Default is false. -I, -\-traffic-ops-insecure - [true | false] ignore certificate errors from Traffic Ops + [true | false] ignore certificate errors from Traffic Ops -k, -\-install-packages - Whether to install necessary packages. Default is false. + Whether to install necessary packages. Default is false. + +-\-local-ats-version + + [true | false] whether to use the local installed ATS version + for config generation. If false, attempt to use the Server + Package Parameter and fall back to ATS 5. If true and the + local ATS version cannot be found, an error will be logged + and the version set to ATS 5. Default is false. -M, -\-maxmind-location=value - URL of a maxmind gzipped database file, to be installed into - the trafficserver etc directory. + URL of a maxmind gzipped database file, to be installed into + the trafficserver etc directory. -m, -\-run-mode=value - [badass | report | revalidate | syncds] run mode. Optional, convenience flag which sets other flags for common usage scenarios. - syncds keeps the defaults: - --report-only=false - --files=all - --install-packages=false - --service-action=reload - --ignore-update-flag=false - --update-ipallow=false - --no-unset-update-flag=false - revalidate sets --files=reval - --wait-for-parents=true - badass sets --install-packages=true - --service-action=restart - --ignore-update-flag=true - --update-ipallow=true - report sets --report-only=true - - Note the 'syncds' settings are all the flag defaults. Hence, if no mode is set, the default is effectively 'syncds'. - - If any of the related flags are also set, they override the mode's default behavior. - - -n, -\-no-cache + [badass | report | revalidate | syncds] run mode. Optional, + convenience flag which sets other flags for common usage + scenarios. + + syncds keeps the defaults: + --report-only=false + --files=all + --install-packages=false + --service-action=reload + --ignore-update-flag=false + --update-ipallow=false + --no-unset-update-flag=false + + revalidate sets --files=reval + --wait-for-parents=true + + badass sets --install-packages=true + --service-action=restart + --ignore-update-flag=true + --update-ipallow=true + + report sets --report-only=true + + Note the 'syncds' settings are all the flag defaults. Hence, + if no mode is set, the default is effectively 'syncds'. + + If any of the related flags are also set, they override the + mode's default behavior. + +-n, -\-no-cache Whether to not use a cache and make conditional requests to Traffic Ops. Default is false: use cache. -o, -\-report-only - Log information about necessary files and actions, but take - no action. Default is false + Log information about necessary files and actions, but take + no action. Default is false -p, -\-reverse-proxy-disable - [false | true] bypass the reverse proxy even if one has been - configured default is false + [false | true] bypass the reverse proxy even if one has been + configured default is false -P, -\-traffic-ops-password=value - Traffic Ops password. Required. May also be set with the - environment variable TO_PASS + Traffic Ops password. Required. May also be set with the + environment variable TO_PASS -r, -\-num-retries=value - [number] retry connection to Traffic Ops URL [number] times, - default is 3 [3] + [number] retry connection to Traffic Ops URL [number] times, + default is 3 [3] -R, -\-trafficserver-home=value - Trafficserver Package directory. May also be set with the - environment variable TS_HOME + Trafficserver Package directory. May also be set with the + environment variable TS_HOME -s, -\-silent - Silent. Errors are not logged, and the 'verbose' flag is - ignored. If a fatal error occurs, the return code will be - non-zero but no text will be output to stderr + Silent. Errors are not logged, and the 'verbose' flag is + ignored. If a fatal error occurs, the return code will be + non-zero but no text will be output to stderr -t, -\-traffic-ops-timeout-milliseconds=value - Timeout in milli-seconds for Traffic Ops requests, default - is 30000 [30000] + Timeout in milli-seconds for Traffic Ops requests, default + is 30000 [30000] -u, -\-traffic-ops-url=value - Traffic Ops URL. Must be the full URL, including the scheme. - Required. May also be set with the environment variable - TO_URL + Traffic Ops URL. Must be the full URL, including the scheme. + Required. May also be set with the environment variable + TO_URL -U, -\-traffic-ops-user=value - Traffic Ops username. Required. May also be set with the - environment variable TO_USER + Traffic Ops username. Required. May also be set with the + environment variable TO_USER -V, -\-default-client-tls-versions=value - Comma-delimited list of default TLS versions for Delivery - Services with no Parameter, e.g. - --default-tls-versions='1.1,1.2,1.3'. If omitted, all - versions are enabled. + Comma-delimited list of default TLS versions for Delivery + Services with no Parameter, e.g. + --default-tls-versions='1.1,1.2,1.3'. If omitted, all + versions are enabled. -v, -\-verbose - Log verbosity. Logging is output to stderr. By default, - errors are logged. To log warnings, pass '-v'. To log info, - pass '-vv'. To omit error logging, see '-s' [0] + Log verbosity. Logging is output to stderr. By default, + errors are logged. To log warnings, pass '-v'. To log info, + pass '-vv'. To omit error logging, see '-s' [0] -W, -\-wait-for-parents - [true | false] do not update if parent_pending = 1 in the - update json. Default is false + [true | false] do not update if parent_pending = 1 in the + update json. Default is false # MODES diff --git a/cache-config/t3c-apply/config/config.go b/cache-config/t3c-apply/config/config.go index 8d335630ac..edae35e8e8 100644 --- a/cache-config/t3c-apply/config/config.go +++ b/cache-config/t3c-apply/config/config.go @@ -26,6 +26,7 @@ import ( "net/url" "os" "os/exec" + "path/filepath" "strings" "time" @@ -116,6 +117,7 @@ type Cfg struct { UpdateIPAllow bool Version string GitRevision string + LocalATSVersion string } func (cfg Cfg) AppVersion() string { return t3cutil.VersionStr(AppName, cfg.Version, cfg.GitRevision) } @@ -278,6 +280,9 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) { const defaultUseStrategies = t3cutil.UseStrategiesFlagFalse useStrategiesPtr := getopt.EnumLong(useStrategiesFlagName, 0, []string{string(t3cutil.UseStrategiesFlagTrue), string(t3cutil.UseStrategiesFlagCore), string(t3cutil.UseStrategiesFlagCore), ""}, "", "[true | core| false] whether to generate config using strategies.yaml instead of parent.config. If true use the parent_select plugin, if 'core' use ATS core strategies, if false use parent.config.") + const useLocalATSVersionFlagName = "local-ats-version" + useLocalATSVersionPtr := getopt.BoolLong(useLocalATSVersionFlagName, 0, "[true | false] whether to use the local installed ATS version for config generation. If false, attempt to use the Server Package Parameter and fall back to ATS 5. If true and the local ATS version cannot be found, an error will be logged and the version set to ATS 5. Default is false") + const runModeFlagName = "run-mode" runModePtr := getopt.StringLong(runModeFlagName, 'm', "", `[badass | report | revalidate | syncds] run mode. Optional, convenience flag which sets other flags for common usage scenarios. syncds keeps the defaults: @@ -475,6 +480,15 @@ If any of the related flags are also set, they override the mode's default behav toInfoLog = append(toInfoLog, fmt.Sprintf("TSHome: %s, TSConfigDir: %s\n", TSHome, tsConfigDir)) } + atsVersionStr := "" + if *useLocalATSVersionPtr { + atsVersionStr, err = GetATSVersionStr(tsHome) + if err != nil { + return Cfg{}, errors.New("getting local ATS version: " + err.Error()) + } + } + toInfoLog = append(toInfoLog, fmt.Sprintf("ATSVersionStr: '%s'\n", atsVersionStr)) + usageStr := "basic usage: t3c-apply --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache" if strings.TrimSpace(toURL) == "" { return Cfg{}, errors.New("Missing required argument --traffic-ops-url or TO_URL environment variable. " + usageStr) @@ -539,6 +553,7 @@ If any of the related flags are also set, they override the mode's default behav NoUnsetUpdateFlag: *noUnsetUpdateFlagPtr, Version: appVersion, GitRevision: gitRevision, + LocalATSVersion: atsVersionStr, } if err = log.InitCfg(cfg); err != nil { @@ -604,6 +619,22 @@ func getOSSvcManagement() SvcManagement { return _svcManager } +func GetATSVersionStr(tsHome string) (string, error) { + tsPath := tsHome + tsPath = filepath.Join(tsPath, "bin") + tsPath = filepath.Join(tsPath, "traffic_server") + + stdOut, stdErr, code := t3cutil.Do(`sh`, `-c`, `set -o pipefail && `+tsPath+` --version | head -1 | awk '{print $3}'`) + if code != 0 { + return "", fmt.Errorf("traffic_server --version returned code %v stderr '%v' stdout '%v'", code, string(stdErr), string(stdOut)) + } + atsVersion := strings.TrimSpace(string(stdOut)) + if atsVersion == "" { + return "", fmt.Errorf("traffic_server --version returned nothing, code %v stderr '%v' stdout '%v'", code, string(stdErr), string(stdOut)) + } + return atsVersion, nil +} + func printConfig(cfg Cfg) { // TODO add new flags log.Debugf("LogLocationDebug: %s\n", cfg.LogLocationDebug) @@ -621,6 +652,7 @@ func printConfig(cfg Cfg) { log.Debugf("TOPass: Pass len: '%d'\n", len(cfg.TOPass)) log.Debugf("TOURL: %s\n", cfg.TOURL) log.Debugf("TSHome: %s\n", TSHome) + log.Debugf("LocalATSVersion: %s\n", cfg.LocalATSVersion) log.Debugf("WaitForParents: %v\n", cfg.WaitForParents) log.Debugf("YumOptions: %s\n", cfg.YumOptions) log.Debugf("MaxmindLocation: %s\n", cfg.MaxMindLocation) diff --git a/cache-config/t3c-apply/torequest/cmd.go b/cache-config/t3c-apply/torequest/cmd.go index d6263292e3..871dcc60c7 100644 --- a/cache-config/t3c-apply/torequest/cmd.go +++ b/cache-config/t3c-apply/torequest/cmd.go @@ -94,6 +94,9 @@ func generate(cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) { if cfg.Files == t3cutil.ApplyFilesFlagReval { args = append(args, "--revalidate-only") } + if cfg.LocalATSVersion != "" { + args = append(args, "--ats-version="+cfg.LocalATSVersion) + } args = append(args, "--via-string-release="+strconv.FormatBool(!cfg.OmitViaStringRelease)) args = append(args, "--no-outgoing-ip="+strconv.FormatBool(cfg.NoOutgoingIP)) args = append(args, "--disable-parent-config-comments="+strconv.FormatBool(cfg.DisableParentConfigComments)) diff --git a/cache-config/t3c-generate/README.md b/cache-config/t3c-generate/README.md index bbcf6c3ff7..db439ba750 100644 --- a/cache-config/t3c-generate/README.md +++ b/cache-config/t3c-generate/README.md @@ -61,6 +61,13 @@ The output is a JSON array of objects containing the file and its metadata. records.config is not serving H2. If omitted, H2 is disabled. +-a, -\-ats-version + + The ATS version, e.g. 9.1.2-42.abc123.el7.x86_64. If omitted + generation will attempt to get the ATS version from the + Server Profile Parameters, and fall back to + lib/go-atscfg.DefaultATSVersion. + -b, -\-dns-local-bind Whether to use the server's Service Addresses to set the ATS diff --git a/cache-config/t3c-generate/cfgfile/all.go b/cache-config/t3c-generate/cfgfile/all.go index eb1c9ef698..8a04c0ab07 100644 --- a/cache-config/t3c-generate/cfgfile/all.go +++ b/cache-config/t3c-generate/cfgfile/all.go @@ -42,7 +42,7 @@ func GetAllConfigs( return nil, errors.New("server hostname is nil") } - configFiles, warnings, err := MakeConfigFilesList(toData, cfg.Dir) + configFiles, warnings, err := MakeConfigFilesList(toData, cfg.Dir, cfg.ATSMajorVersion) logWarnings("generating config files list: ", warnings) if err != nil { return nil, errors.New("creating meta: " + err.Error()) diff --git a/cache-config/t3c-generate/cfgfile/wrappers.go b/cache-config/t3c-generate/cfgfile/wrappers.go index 3f908ba0c0..894d58e6ea 100644 --- a/cache-config/t3c-generate/cfgfile/wrappers.go +++ b/cache-config/t3c-generate/cfgfile/wrappers.go @@ -26,16 +26,16 @@ import ( "github.com/apache/trafficcontrol/lib/go-tc" ) -// // This file has wrappers that turn lib/go-atscfg Make funcs into ConfigFileFunc types. // // We don't want to make lib/go-atscfg functions take a TOData, because then users wanting to generate a single file would have to fetch all kinds of data that file doesn't need, or else pass objects they know it doesn't currently need as nil and risk it crashing if that func is changed to use it in the future. // // But it's useful to map filenames to functions for dispatch. Hence these wrappers. // - +// The atsMajorVersion may be 0 to default to the Server Package Parameter. +// // MakeConfigFilesList returns the list of config files, any warnings, and any error. -func MakeConfigFilesList(toData *t3cutil.ConfigData, dir string) ([]atscfg.CfgMeta, []string, error) { +func MakeConfigFilesList(toData *t3cutil.ConfigData, dir string, atsMajorVersion uint) ([]atscfg.CfgMeta, []string, error) { configFiles, warnings, err := atscfg.MakeConfigFilesList( dir, toData.Server, @@ -45,7 +45,9 @@ func MakeConfigFilesList(toData *t3cutil.ConfigData, dir string) ([]atscfg.CfgMe toData.GlobalParams, toData.CacheGroups, toData.Topologies, - &atscfg.ConfigFilesListOpts{}, + &atscfg.ConfigFilesListOpts{ + ATSMajorVersion: atsMajorVersion, + }, ) return configFiles, warnings, err } @@ -119,8 +121,14 @@ func MakeLoggingDotConfig(toData *t3cutil.ConfigData, fileName string, hdrCommen } func MakeLoggingDotYAML(toData *t3cutil.ConfigData, fileName string, hdrCommentTxt string, cfg config.Cfg) (atscfg.Cfg, error) { - opts := &atscfg.LoggingDotYAMLOpts{HdrComment: hdrCommentTxt} - return atscfg.MakeLoggingDotYAML(toData.Server, toData.ServerParams, opts) + return atscfg.MakeLoggingDotYAML( + toData.Server, + toData.ServerParams, + &atscfg.LoggingDotYAMLOpts{ + HdrComment: hdrCommentTxt, + ATSMajorVersion: cfg.ATSMajorVersion, + }, + ) } func MakeSSLServerNameYAML(toData *t3cutil.ConfigData, fileName string, hdrCommentTxt string, cfg config.Cfg) (atscfg.Cfg, error) { @@ -188,8 +196,9 @@ func MakeParentDotConfig(toData *t3cutil.ConfigData, fileName string, hdrComment toData.DeliveryServiceServers, toData.CDN, &atscfg.ParentConfigOpts{ - HdrComment: hdrCommentTxt, - AddComments: cfg.ParentComments, // TODO add a CLI flag? + HdrComment: hdrCommentTxt, + AddComments: cfg.ParentComments, // TODO add a CLI flag? + ATSMajorVersion: cfg.ATSMajorVersion, }, ) } @@ -239,6 +248,7 @@ func MakeRemapDotConfig(toData *t3cutil.ConfigData, fileName string, hdrCommentT VerboseComments: true, UseStrategies: cfg.UseStrategies == t3cutil.UseStrategiesFlagTrue || cfg.UseStrategies == t3cutil.UseStrategiesFlagCore, UseStrategiesCore: cfg.UseStrategies == t3cutil.UseStrategiesFlagCore, + ATSMajorVersion: cfg.ATSMajorVersion, }, ) } @@ -264,7 +274,6 @@ func MakeVolumeDotConfig(toData *t3cutil.ConfigData, fileName string, hdrComment } func MakeHeaderRewrite(toData *t3cutil.ConfigData, fileName string, hdrCommentTxt string, cfg config.Cfg) (atscfg.Cfg, error) { - opts := &atscfg.HeaderRewriteDotConfigOpts{HdrComment: hdrCommentTxt} return atscfg.MakeHeaderRewriteDotConfig( fileName, toData.DeliveryServices, @@ -276,7 +285,10 @@ func MakeHeaderRewrite(toData *t3cutil.ConfigData, fileName string, hdrCommentTx toData.ServerCapabilities, toData.DSRequiredCapabilities, toData.Topologies, - opts, + &atscfg.HeaderRewriteDotConfigOpts{ + HdrComment: hdrCommentTxt, + ATSMajorVersion: cfg.ATSMajorVersion, + }, ) } @@ -315,6 +327,7 @@ func MakeStrategiesDotYAML(toData *t3cutil.ConfigData, fileName string, hdrComme &atscfg.StrategiesYAMLOpts{ HdrComment: hdrCommentTxt, VerboseComments: cfg.ParentComments, // TODO add a CLI flag? + ATSMajorVersion: cfg.ATSMajorVersion, }, ) } diff --git a/cache-config/t3c-generate/config/config.go b/cache-config/t3c-generate/config/config.go index 765df743dc..338c0889c5 100644 --- a/cache-config/t3c-generate/config/config.go +++ b/cache-config/t3c-generate/config/config.go @@ -55,6 +55,7 @@ type Cfg struct { ViaRelease bool SetDNSLocalBind bool NoOutgoingIP bool + ATSMajorVersion uint ParentComments bool DefaultEnableH2 bool DefaultTLSVersions []atscfg.TLSVersion @@ -83,6 +84,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) { defaultEnableH2 := getopt.BoolLong("default-client-enable-h2", '2', "Whether to enable HTTP/2 on Delivery Services by default, if they have no explicit Parameter. This is irrelevant if ATS records.config is not serving H2. If omitted, H2 is disabled.") defaultTLSVersionsStr := getopt.StringLong("default-client-tls-versions", 'T', "", "Comma-delimited list of default TLS versions for Delivery Services with no Parameter, e.g. '--default-tls-versions=1.1,1.2,1.3'. If omitted, all versions are enabled.") noOutgoingIP := getopt.BoolLong("no-outgoing-ip", 'i', "Whether to not set the records.config outgoing IP to the server's addresses in Traffic Ops. Default is false.") + atsVersion := getopt.StringLong("ats-version", 'a', "", "The ATS version, e.g. 9.1.2-42.abc123.el7.x86_64. If omitted, generation will attempt to get the ATS version from the Server Parameters, and fall back to lib/go-atscfg.DefaultATSVersion") verbosePtr := getopt.CounterLong("verbose", 'v', `Log verbosity. Logging is output to stderr. By default, errors are logged. To log warnings, pass '-v'. To log info, pass '-vv'. To omit error logging, see '-s'`) silentPtr := getopt.BoolLong("silent", 's', `Silent. Errors are not logged, and the 'verbose' flag is ignored. If a fatal error occurs, the return code will be non-zero but no text will be output to stderr`) @@ -123,6 +125,17 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) { return Cfg{}, errors.New("Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)") } + // The flag takes the full version, for forward-compatibility in case we need it in the future, + // but we only need the major version at the moment. + atsMajorVersion := uint(0) + if *atsVersion != "" { + err := error(nil) + atsMajorVersion, err = atscfg.GetATSMajorVersionFromATSVersion(*atsVersion) + if err != nil { + return Cfg{}, errors.New("parsing ATS version '" + *atsVersion + "': " + err.Error()) + } + } + defaultTLSVersions := atscfg.DefaultDefaultTLSVersions *defaultTLSVersionsStr = strings.TrimSpace(*defaultTLSVersionsStr) @@ -154,6 +167,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) { ViaRelease: *viaRelease, SetDNSLocalBind: *dnsLocalBind, NoOutgoingIP: *noOutgoingIP, + ATSMajorVersion: atsMajorVersion, ParentComments: !(*disableParentConfigComments), DefaultEnableH2: *defaultEnableH2, DefaultTLSVersions: defaultTLSVersions, diff --git a/lib/go-atscfg/atscfg.go b/lib/go-atscfg/atscfg.go index 92abf8a09e..45287c3f91 100644 --- a/lib/go-atscfg/atscfg.go +++ b/lib/go-atscfg/atscfg.go @@ -226,21 +226,21 @@ func makeHdrComment(hdrComment string) string { return "# " + hdrComment + "\n\n" } -// getATSMajorVersionFromATSVersion returns the major version of the given profile's package trafficserver parameter. +// GetATSMajorVersionFromATSVersion returns the major version of the given profile's package trafficserver parameter. // The atsVersion is typically a Parameter on the Server's Profile, with the configFile "package" name "trafficserver". // Returns an error if atsVersion is empty or does not start with an unsigned integer followed by a period or nothing. -func getATSMajorVersionFromATSVersion(atsVersion string) (int, error) { +func GetATSMajorVersionFromATSVersion(atsVersion string) (uint, error) { dotPos := strings.Index(atsVersion, ".") if dotPos == -1 { dotPos = len(atsVersion) // if there's no '.' then assume the whole string is just a major version. } majorVerStr := atsVersion[:dotPos] - majorVer, err := strconv.Atoi(majorVerStr) - if err != nil || majorVer < 0 { + majorVer, err := strconv.ParseUint(majorVerStr, 10, 64) + if err != nil || majorVer == 0 || majorVer > 9999 { return 0, errors.New("unexpected version format '" + majorVerStr + "', expected e.g. '7.1.2.whatever'") } - return majorVer, nil + return uint(majorVer), nil } // genericProfileConfig generates a generic profile config text, from the profile's parameters with the given config file name. @@ -617,10 +617,28 @@ func getServiceAddresses(sv *Server) (net.IP, net.IP) { return v4, v6 } -// getATSMajorVersion returns the ATS major version from the config_file 'package' name 'trafficserver' Parameter on the given Server Profile Parameters. +// getATSMajorVersion takes a config variable of the version, the Server Parameters, and a pointer to warnings to populate. +// This allows callers to use the config variable if it was given, or get the ATS version from the Server Parameters if it wasn't, and add to a warnings variable, in a single line. +// +// If more flexibility is needed, getATSMajorVersionFromParams may be called directly; +// but it should generally be avoided, functions should always take a config variable for the +// ATS version, in case a user wants to manage the ATS package outside ATC. +func getATSMajorVersion(atsMajorVersion uint, serverParams []tc.Parameter, warnings *[]string) uint { + if atsMajorVersion != 0 { + return atsMajorVersion + } + verWarns := []string{} + atsMajorVersion, verWarns = getATSMajorVersionFromParams(serverParams) + *warnings = append(*warnings, verWarns...) + return atsMajorVersion +} + +// getATSMajorVersionFromParams should generally not be called directly. Rather, functions should always take the version as a config parameter which may be omitted, and call getATSMajorVersion. +// +// It returns the ATS major version from the config_file 'package' name 'trafficserver' Parameter on the given Server Profile Parameters. // If no Parameter is found, or the value is malformed, a warning or error is logged and DefaultATSVersion is returned. // Returns the ATS major version, and any warnings -func getATSMajorVersion(serverParams []tc.Parameter) (int, []string) { +func getATSMajorVersionFromParams(serverParams []tc.Parameter) (uint, []string) { warnings := []string{} atsVersionParam := "" for _, param := range serverParams { @@ -635,10 +653,10 @@ func getATSMajorVersion(serverParams []tc.Parameter) (int, []string) { atsVersionParam = DefaultATSVersion } - atsMajorVer, err := getATSMajorVersionFromATSVersion(atsVersionParam) + atsMajorVer, err := GetATSMajorVersionFromATSVersion(atsVersionParam) if err != nil { warnings = append(warnings, "getting ATS major version from server Profile Parameter, using default: "+err.Error()) - atsMajorVer, err = getATSMajorVersionFromATSVersion(DefaultATSVersion) + atsMajorVer, err = GetATSMajorVersionFromATSVersion(DefaultATSVersion) if err != nil { // should never happen warnings = append(warnings, "getting ATS major version from default version! Should never happen! Using 0, config will be malformed! : "+err.Error()) diff --git a/lib/go-atscfg/atscfg_test.go b/lib/go-atscfg/atscfg_test.go index b25c04dbc6..db0f6bb523 100644 --- a/lib/go-atscfg/atscfg_test.go +++ b/lib/go-atscfg/atscfg_test.go @@ -76,7 +76,7 @@ func TestTrimParamUnderscoreNumSuffix(t *testing.T) { } func TestGetATSMajorVersionFromATSVersion(t *testing.T) { - inputExpected := map[string]int{ + inputExpected := map[string]uint{ `7.1.2-34.56abcde.el7.centos.x86_64`: 7, `8`: 8, `8.1`: 8, @@ -98,14 +98,14 @@ func TestGetATSMajorVersionFromATSVersion(t *testing.T) { } for input, expected := range inputExpected { - if actual, err := getATSMajorVersionFromATSVersion(input); err != nil { + if actual, err := GetATSMajorVersionFromATSVersion(input); err != nil { t.Errorf("expected %v actual: error '%v'", expected, err) } else if actual != expected { t.Errorf("expected %v actual: %v", expected, actual) } } for _, input := range errExpected { - if actual, err := getATSMajorVersionFromATSVersion(input); err == nil { + if actual, err := GetATSMajorVersionFromATSVersion(input); err == nil { t.Errorf("input %v expected: error, actual: nil error '%v'", input, actual) } } diff --git a/lib/go-atscfg/headerrewritedotconfig.go b/lib/go-atscfg/headerrewritedotconfig.go index ff1c509e62..7cadd0bd06 100644 --- a/lib/go-atscfg/headerrewritedotconfig.go +++ b/lib/go-atscfg/headerrewritedotconfig.go @@ -63,6 +63,14 @@ type HeaderRewriteDotConfigOpts struct { // This should be the text desired, without comment syntax (like # or //). The file's comment syntax will be added. // To omit the header comment, pass the empty string. HdrComment string + + // ATSMajorVersion is the integral major version of Apache Traffic server, + // used to generate the proper config for the proper version. + // + // If omitted or 0, the major version will be read from the Server's Profile Parameter config file 'package' name 'trafficserver'. If no such Parameter exists, the ATS version will default to 5. + // This was the old Traffic Control behavior, before the version was specifiable externally. + // + ATSMajorVersion uint } // MakeHeaderRewriteDotConfig makes the header rewrite file for @@ -151,8 +159,7 @@ func MakeHeaderRewriteDotConfig( atsRqstMaxHdrSize, paramWarns := getMaxRequestHeaderParam(tcServerParams) warnings = append(warnings, paramWarns...) - atsMajorVersion, verWarns := getATSMajorVersion(tcServerParams) - warnings = append(warnings, verWarns...) + atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, tcServerParams, &warnings) assignedTierPeers, assignWarns := getAssignedTierPeers(server, ds, topology, servers, deliveryServiceServers, cacheGroupsArr, serverCapabilities, requiredCapabilities[*ds.ID]) warnings = append(warnings, assignWarns...) @@ -530,13 +537,13 @@ var returnRe = regexp.MustCompile(`\s*__RETURN__\s*`) // If they're placed after, custom rewrites with [L] directives will result in them being applied inconsistently and incorrectly. // // The headerRewriteTxt is the custom header rewrite from the Delivery Service. This should be used for any logic that depends on it. The various header rewrite fields (EdgeHeaderRewrite, InnerHeaderRewrite, etc should never be used inside this function, since this function doesn't know what tier the server is at. This function should not insert the headerRewriteText, but may use it to make decisions about what to insert. -func makeATCHeaderRewriteDirectives(ds *DeliveryService, headerRewriteTxt *string, serverIsLastTier bool, numLastTierServers int, atsMajorVersion int, atsRqstMaxHdrSize int) string { +func makeATCHeaderRewriteDirectives(ds *DeliveryService, headerRewriteTxt *string, serverIsLastTier bool, numLastTierServers int, atsMajorVersion uint, atsRqstMaxHdrSize int) string { return makeATCHeaderRewriteDirectiveMaxOriginConns(ds, headerRewriteTxt, serverIsLastTier, numLastTierServers, atsMajorVersion) + makeATCHeaderRewriteDirectiveServiceCategoryHdr(ds, headerRewriteTxt) + makeATCHeaderRewriteDirectiveMaxRequestHeaderSize(ds, serverIsLastTier, atsRqstMaxHdrSize) } // makeATCHeaderRewriteDirectiveMaxOriginConns generates the Max Origin Connections header rewrite text, which may be empty. -func makeATCHeaderRewriteDirectiveMaxOriginConns(ds *DeliveryService, headerRewriteTxt *string, serverIsLastTier bool, numLastTierServers int, atsMajorVersion int) string { +func makeATCHeaderRewriteDirectiveMaxOriginConns(ds *DeliveryService, headerRewriteTxt *string, serverIsLastTier bool, numLastTierServers int, atsMajorVersion uint) string { if !serverIsLastTier || (ds.MaxOriginConnections == nil || *ds.MaxOriginConnections < 1) || numLastTierServers < 1 { diff --git a/lib/go-atscfg/loggingdotyaml.go b/lib/go-atscfg/loggingdotyaml.go index a675907d10..359a1ea3ea 100644 --- a/lib/go-atscfg/loggingdotyaml.go +++ b/lib/go-atscfg/loggingdotyaml.go @@ -37,15 +37,23 @@ type LoggingDotYAMLOpts struct { // This should be the text desired, without comment syntax (like # or //). The file's comment syntax will be added. // To omit the header comment, pass the empty string. HdrComment string + + // ATSMajorVersion is the integral major version of Apache Traffic server, + // used to generate the proper config for the proper version. + // + // If omitted or 0, the major version will be read from the Server's Profile Parameter config file 'package' name 'trafficserver'. If no such Parameter exists, the ATS version will default to 5. + // This was the old Traffic Control behavior, before the version was specifiable externally. + // + ATSMajorVersion uint } func MakeLoggingDotYAML( server *Server, serverParams []tc.Parameter, - opts *LoggingDotYAMLOpts, + opt *LoggingDotYAMLOpts, ) (Cfg, error) { - if opts == nil { - opts = &LoggingDotYAMLOpts{} + if opt == nil { + opt = &LoggingDotYAMLOpts{} } warnings := []string{} requiredIndent := 0 @@ -61,15 +69,15 @@ func MakeLoggingDotYAML( paramData, paramWarns := paramsToMap(filterParams(serverParams, LoggingYAMLFileName, "", "", "location")) warnings = append(warnings, paramWarns...) - hdr := makeHdrComment(opts.HdrComment) + hdr := makeHdrComment(opt.HdrComment) + + atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, serverParams, &warnings) - version, vWarn := getATSMajorVersion(serverParams) - warnings = append(warnings, vWarn...) // note we use the same const as logs.xml - this isn't necessarily a requirement, and we may want to make separate variables in the future. maxLogObjects := MaxLogObjects text := hdr - if version >= 9 { + if atsMajorVersion >= 9 { text += "\nlogging:" requiredIndent += 2 } diff --git a/lib/go-atscfg/meta.go b/lib/go-atscfg/meta.go index cbd23aa99d..aac7b7df45 100644 --- a/lib/go-atscfg/meta.go +++ b/lib/go-atscfg/meta.go @@ -34,6 +34,13 @@ type CfgMeta struct { // ConfigFilesListOpts contains settings to configure generation options. type ConfigFilesListOpts struct { + // ATSMajorVersion is the integral major version of Apache Traffic server, + // used to generate the proper config for the proper version. + // + // If omitted or 0, the major version will be read from the Server's Profile Parameter config file 'package' name 'trafficserver'. If no such Parameter exists, the ATS version will default to 5. + // This was the old Traffic Control behavior, before the version was specifiable externally. + // + ATSMajorVersion uint } // MakeMetaObj returns the list of config files, any warnings, and any errors. @@ -71,8 +78,7 @@ func MakeConfigFilesList( return nil, warnings, errors.New("server missing Profile") } - atsMajorVer, verWarns := getATSMajorVersion(serverParams) - warnings = append(warnings, verWarns...) + atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, serverParams, &warnings) dses, dsWarns := filterConfigFileDSes(server, deliveryServices, deliveryServiceServers) warnings = append(warnings, dsWarns...) @@ -129,7 +135,7 @@ locationParamsFor: configFiles = append(configFiles, atsCfg) } - configFiles, configDirWarns, err := addMetaObjConfigDir(configFiles, configDir, server, locationParams, uriSignedDSes, dses, cacheGroupArr, topologies, atsMajorVer) + configFiles, configDirWarns, err := addMetaObjConfigDir(configFiles, configDir, server, locationParams, uriSignedDSes, dses, cacheGroupArr, topologies, atsMajorVersion) warnings = append(warnings, configDirWarns...) return configFiles, warnings, err } @@ -148,7 +154,7 @@ func addMetaObjConfigDir( dses map[tc.DeliveryServiceName]DeliveryService, cacheGroupArr []tc.CacheGroupNullable, topologies []tc.Topology, - atsMajorVer int, + atsMajorVersion uint, ) ([]CfgMeta, []string, error) { warnings := []string{} @@ -171,7 +177,7 @@ func addMetaObjConfigDir( // If they don't exist, create them. // If they exist with a relative path, prepend configDir. // If any exist with a relative path, or don't exist, and configDir is empty, return an error. - for _, fileName := range requiredFiles(atsMajorVer) { + for _, fileName := range requiredFiles(atsMajorVersion) { if _, ok := configFilesM[fileName]; ok { continue } @@ -413,8 +419,8 @@ type configProfileParams struct { Path string } -func requiredFiles(atsMajorVer int) []string { - if atsMajorVer >= 9 { +func requiredFiles(atsMajorVersion uint) []string { + if atsMajorVersion >= 9 { return requiredFiles9() } return requiredFiles8() diff --git a/lib/go-atscfg/parentabstraction.go b/lib/go-atscfg/parentabstraction.go index c79310b268..dab87aa27f 100644 --- a/lib/go-atscfg/parentabstraction.go +++ b/lib/go-atscfg/parentabstraction.go @@ -273,7 +273,7 @@ var DefaultUnavailableServerRetryCodes = []int{503} const DefaultIgnoreQueryStringInParentSelection = false -func parentAbstractionToParentDotConfig(pa *ParentAbstraction, opt *ParentConfigOpts, atsMajorVersion int) (string, []string, error) { +func parentAbstractionToParentDotConfig(pa *ParentAbstraction, opt *ParentConfigOpts, atsMajorVersion uint) (string, []string, error) { warnings := []string{} txt := "" @@ -301,7 +301,7 @@ func parentAbstractionToParentDotConfig(pa *ParentAbstraction, opt *ParentConfig return txt, warnings, nil } -func (svc *ParentAbstractionService) ToParentDotConfigLine(opt *ParentConfigOpts, atsMajorVersion int) (string, []string, error) { +func (svc *ParentAbstractionService) ToParentDotConfigLine(opt *ParentConfigOpts, atsMajorVersion uint) (string, []string, error) { warnings := []string{} txt := "" if opt.AddComments && svc.Comment != "" { @@ -371,7 +371,7 @@ func (svc *ParentAbstractionService) ToParentDotConfigLine(opt *ParentConfigOpts if atsMajorVersion >= 9 { txt += ` simple_server_retry_responses="` + strings.Join(intsToStrs(svc.ErrorResponseCodes), `,`) + `"` } else { - warnings = append(warnings, "Service '"+svc.Name+"' had simple retry codes '"+strings.Join(intsToStrs(svc.ErrorResponseCodes), ",")+"' but ATS version "+strconv.Itoa(atsMajorVersion)+" < 9 does not support custom simple retry codes, omitting!") + warnings = append(warnings, "Service '"+svc.Name+"' had simple retry codes '"+strings.Join(intsToStrs(svc.ErrorResponseCodes), ",")+"' but ATS version "+strconv.FormatUint(uint64(atsMajorVersion), 10)+" < 9 does not support custom simple retry codes, omitting!") } } diff --git a/lib/go-atscfg/parentdotconfig.go b/lib/go-atscfg/parentdotconfig.go index 8eabf1a20b..2de9d97f8f 100644 --- a/lib/go-atscfg/parentdotconfig.go +++ b/lib/go-atscfg/parentdotconfig.go @@ -100,6 +100,14 @@ type ParentConfigOpts struct { // This should be the text desired, without comment syntax (like # or //). The file's comment syntax will be added. // To omit the header comment, pass the empty string. HdrComment string + + // ATSMajorVersion is the integral major version of Apache Traffic server, + // used to generate the proper config for the proper version. + // + // If omitted or 0, the major version will be read from the Server's Profile Parameter config file 'package' name 'trafficserver'. If no such Parameter exists, the ATS version will default to 5. + // This was the old Traffic Control behavior, before the version was specifiable externally. + // + ATSMajorVersion uint } func MakeParentDotConfig( @@ -116,7 +124,10 @@ func MakeParentDotConfig( cdn *tc.CDN, opt *ParentConfigOpts, ) (Cfg, error) { - parentAbstraction, warnings, err := makeParentDotConfigData( + warnings := []string{} + atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, tcServerParams, &warnings) + + parentAbstraction, dataWarns, err := makeParentDotConfigData( dses, server, servers, @@ -129,15 +140,14 @@ func MakeParentDotConfig( dss, cdn, opt, + atsMajorVersion, ) + warnings = append(warnings, dataWarns...) if err != nil { return Cfg{}, makeErr(warnings, err.Error()) } - atsMajorVer, verWarns := getATSMajorVersion(tcServerParams) - warnings = append(warnings, verWarns...) - - text, paWarns, err := parentAbstractionToParentDotConfig(parentAbstraction, opt, atsMajorVer) + text, paWarns, err := parentAbstractionToParentDotConfig(parentAbstraction, opt, atsMajorVersion) warnings = append(warnings, paWarns...) if err != nil { return Cfg{}, makeErr(warnings, err.Error()) @@ -169,6 +179,7 @@ func makeParentDotConfigData( dss []DeliveryServiceServer, cdn *tc.CDN, opt *ParentConfigOpts, + atsMajorVersion uint, ) (*ParentAbstraction, []string, error) { if opt == nil { opt = &ParentConfigOpts{} @@ -188,10 +199,6 @@ func makeParentDotConfigData( return nil, warnings, errors.New("server TCPPort missing") } - // TODO remove, the abstraction shouldn't depend on the ATS version - atsMajorVer, verWarns := getATSMajorVersion(tcServerParams) - warnings = append(warnings, verWarns...) - cacheGroups, err := makeCGMap(cacheGroupArr) if err != nil { return nil, warnings, errors.New("making CacheGroup map: " + err.Error()) @@ -393,7 +400,7 @@ func makeParentDotConfigData( cacheGroups, profileParentConfigParams, isMSO, - atsMajorVer, + atsMajorVersion, dsOrigins[DeliveryServiceID(*ds.ID)], opt.AddComments, ) @@ -499,7 +506,7 @@ func makeParentDotConfigData( warnings = append(warnings, "DS "+*ds.XMLID+" has no parent servers") } - parents, secondaryParents, secondaryMode, parentWarns := getMSOParentStrs(&ds, parentInfos[OriginHost(orgURI.Hostname())], atsMajorVer, dsParams.Algorithm, dsParams.TryAllPrimariesBeforeSecondary) + parents, secondaryParents, secondaryMode, parentWarns := getMSOParentStrs(&ds, parentInfos[OriginHost(orgURI.Hostname())], atsMajorVersion, dsParams.Algorithm, dsParams.TryAllPrimariesBeforeSecondary) warnings = append(warnings, parentWarns...) pasvc.Parents = parents pasvc.SecondaryParents = secondaryParents @@ -510,7 +517,7 @@ func makeParentDotConfigData( // textLine += parents + secondaryParents + ` round_robin=` + dsParams.Algorithm + ` qstring=` + parentQStr + ` go_direct=false parent_is_proxy=false` - prWarns := dsParams.FillParentSvcRetries(cacheIsTopLevel, atsMajorVer, pasvc) + prWarns := dsParams.FillParentSvcRetries(cacheIsTopLevel, atsMajorVersion, pasvc) warnings = append(warnings, prWarns...) parentAbstraction.Services = append(parentAbstraction.Services, pasvc) @@ -526,7 +533,7 @@ func makeParentDotConfigData( goDirect := false // goDirect := `go_direct=false` - parents, secondaryParents, secondaryMode, parentWarns := getParentStrs(&ds, dsRequiredCapabilities, parentInfos[deliveryServicesAllParentsKey], atsMajorVer, dsParams.TryAllPrimariesBeforeSecondary) + parents, secondaryParents, secondaryMode, parentWarns := getParentStrs(&ds, dsRequiredCapabilities, parentInfos[deliveryServicesAllParentsKey], atsMajorVersion, dsParams.TryAllPrimariesBeforeSecondary) warnings = append(warnings, parentWarns...) pasvc := &ParentAbstractionService{} @@ -624,7 +631,7 @@ func makeParentDotConfigData( // text += `dest_domain=` + orgURI.Hostname() + ` port=` + orgURI.Port() + ` ` + parents + ` ` + secondaryParents + ` ` + roundRobin + ` ` + goDirect + ` qstring=` + parentQStr + "\n" } - prWarns := dsParams.FillParentSvcRetries(cacheIsTopLevel, atsMajorVer, pasvc) + prWarns := dsParams.FillParentSvcRetries(cacheIsTopLevel, atsMajorVersion, pasvc) warnings = append(warnings, prWarns...) parentAbstraction.Services = append(parentAbstraction.Services, pasvc) @@ -641,7 +648,7 @@ func makeParentDotConfigData( invalidDS := &DeliveryService{} invalidDS.ID = util.IntPtr(-1) tryAllPrimariesBeforeSecondary := false - parents, secondaryParents, secondaryMode, parentWarns := getParentStrs(invalidDS, dsRequiredCapabilities, parentInfos[deliveryServicesAllParentsKey], atsMajorVer, tryAllPrimariesBeforeSecondary) + parents, secondaryParents, secondaryMode, parentWarns := getParentStrs(invalidDS, dsRequiredCapabilities, parentInfos[deliveryServicesAllParentsKey], atsMajorVersion, tryAllPrimariesBeforeSecondary) warnings = append(warnings, parentWarns...) defaultDestText.DestDomain = `.` @@ -1001,7 +1008,7 @@ func getTopologyParentConfigLine( cacheGroups map[tc.CacheGroupName]tc.CacheGroupNullable, profileParentConfigParams map[string]map[string]string, isMSO bool, - atsMajorVer int, + atsMajorVersion uint, dsOrigins map[ServerID]struct{}, addComments bool, ) (*ParentAbstractionService, []string, error) { @@ -1070,7 +1077,7 @@ func getTopologyParentConfigLine( pasvc.SecondaryParents = secondaryParents // txt += ` secondary_parent="` + strings.Join(secondaryParents, `;`) + `"` - secondaryModeStr, secondaryModeWarnings := getSecondaryModeStr(dsParams.TryAllPrimariesBeforeSecondary, atsMajorVer, tc.DeliveryServiceName(*ds.XMLID)) + secondaryModeStr, secondaryModeWarnings := getSecondaryModeStr(dsParams.TryAllPrimariesBeforeSecondary, atsMajorVersion, tc.DeliveryServiceName(*ds.XMLID)) warnings = append(warnings, secondaryModeWarnings...) // txt += secondaryModeStr pasvc.SecondaryMode = secondaryModeStr // TODO convert @@ -1103,7 +1110,7 @@ func getTopologyParentConfigLine( // txt += getTopologyParentIsProxyStr(serverPlacement.IsLastCacheTier) // TODO convert - prWarns := dsParams.FillParentSvcRetries(serverPlacement.IsLastCacheTier, atsMajorVer, pasvc) + prWarns := dsParams.FillParentSvcRetries(serverPlacement.IsLastCacheTier, atsMajorVersion, pasvc) warnings = append(warnings, prWarns...) // txt += getParentRetryStr(serverPlacement.IsLastCacheTier, atsMajorVer, dsParams.ParentRetry, dsParams.UnavailableServerRetryResponses, dsParams.MaxSimpleRetries, dsParams.MaxUnavailableServerRetries) @@ -1120,7 +1127,7 @@ func getTopologyParentConfigLine( // // Returns the MaxSimpleRetries, MaxMarkdownRetries, ErrorREsponseCodes, MarkdownResponseCodes. // -// If atsMajorVer < 6, "" is returned (ATS 5 and below don't support retry directives). +// If atsMajorVersion < 6, "" is returned (ATS 5 and below don't support retry directives). // If isLastCacheTier is false, "" is returned. This argument exists to simplify usage. // If parentRetry is "", "" is returned (because the other directives are unused if parent_retry doesn't exist). This is allowed to simplify usage. // If unavailableServerRetryResponses is not "", it must be valid. Use unavailableServerRetryResponsesValid to check. @@ -1128,11 +1135,11 @@ func getTopologyParentConfigLine( // If maxUnavailableServerRetries is "", ParentConfigDSParamDefaultMaxUnavailableServerRetries will be used. // // Does not return errors. If any input is malformed, warnings are returned and that value is set to -1. -func (dsparams parentDSParams) FillParentSvcRetries(isLastCacheTier bool, atsMajorVer int, pasvc *ParentAbstractionService) []string { +func (dsparams parentDSParams) FillParentSvcRetries(isLastCacheTier bool, atsMajorVersion uint, pasvc *ParentAbstractionService) []string { warnings := []string{} if !dsparams.HasRetryParams || // allow parentRetry to be empty, to simplify usage. - atsMajorVer < 6 { // ATS 5 and below don't support parent_retry directives + atsMajorVersion < 6 { // ATS 5 and below don't support parent_retry directives // warnings = append(warnings, "ATS 5 doesn't support parent retry, not using parent retry values") pasvc.MaxSimpleRetries = -1 pasvc.MaxMarkdownRetries = -1 @@ -1210,13 +1217,13 @@ func (dsparams parentDSParams) FillParentSvcRetries(isLastCacheTier bool, atsMaj } // getSecondaryModeStr returns the secondary_mode string, and any warnings. -func getSecondaryModeStr(tryAllPrimariesBeforeSecondary bool, atsMajorVer int, ds tc.DeliveryServiceName) (ParentAbstractionServiceParentSecondaryMode, []string) { +func getSecondaryModeStr(tryAllPrimariesBeforeSecondary bool, atsMajorVersion uint, ds tc.DeliveryServiceName) (ParentAbstractionServiceParentSecondaryMode, []string) { warnings := []string{} if !tryAllPrimariesBeforeSecondary { return ParentAbstractionServiceParentSecondaryModeDefault, warnings } - if atsMajorVer < 8 { - warnings = append(warnings, "DS '"+string(ds)+"' had Parameter "+ParentConfigRetryKeysDefault.SecondaryMode+" but this cache is "+strconv.Itoa(atsMajorVer)+" and secondary_mode isn't supported in ATS until 8. Not using!") + if atsMajorVersion < 8 { + warnings = append(warnings, "DS '"+string(ds)+"' had Parameter "+ParentConfigRetryKeysDefault.SecondaryMode+" but this cache is "+strconv.FormatUint(uint64(atsMajorVersion), 10)+" and secondary_mode isn't supported in ATS until 8. Not using!") return ParentAbstractionServiceParentSecondaryModeDefault, warnings } @@ -1527,7 +1534,7 @@ func getParentStrs( ds *DeliveryService, dsRequiredCapabilities map[int]map[ServerCapability]struct{}, parentInfos []parentInfo, - atsMajorVer int, + atsMajorVersion uint, tryAllPrimariesBeforeSecondary bool, ) ([]*ParentAbstractionServiceParent, []*ParentAbstractionServiceParent, ParentAbstractionServiceParentSecondaryMode, []string) { warnings := []string{} @@ -1568,10 +1575,10 @@ func getParentStrs( // secondaryParents := "" // "secparents" in Perl // TODO the abstract->text needs to take this into account - // if atsMajorVer >= 6 && len(secondaryParentInfo) > 0 { + // if atsMajorVersion >= 6 && len(secondaryParentInfo) > 0 { // parents = `parent="` + strings.Join(parentInfo, "") + `"` // secondaryParents = ` secondary_parent="` + strings.Join(secondaryParentInfo, "") + `"` - secondaryMode, secondaryModeWarnings := getSecondaryModeStr(tryAllPrimariesBeforeSecondary, atsMajorVer, dsName) + secondaryMode, secondaryModeWarnings := getSecondaryModeStr(tryAllPrimariesBeforeSecondary, atsMajorVersion, dsName) warnings = append(warnings, secondaryModeWarnings...) // secondaryParents += secondaryModeStr // } else { @@ -1585,7 +1592,7 @@ func getParentStrs( func getMSOParentStrs( ds *DeliveryService, parentInfos []parentInfo, - atsMajorVer int, + atsMajorVersion uint, msoAlgorithm ParentAbstractionServiceRetryPolicy, tryAllPrimariesBeforeSecondary bool, ) ([]*ParentAbstractionServiceParent, []*ParentAbstractionServiceParent, ParentAbstractionServiceParentSecondaryMode, []string) { @@ -1640,10 +1647,10 @@ func getMSOParentStrs( // secondaryParents := "" // TODO add this logic to the abstraction->text converter - // if atsMajorVer >= 6 && msoAlgorithm == "consistent_hash" && len(secondaryParentStr) > 0 { + // if atsMajorVersion >= 6 && msoAlgorithm == "consistent_hash" && len(secondaryParentStr) > 0 { // parents = `parent="` + strings.Join(parentInfoTxt, "") + `"` // secondaryParents = ` secondary_parent="` + secondaryParentStr + `"` - secondaryMode, secondaryModeWarnings := getSecondaryModeStr(tryAllPrimariesBeforeSecondary, atsMajorVer, dsName) + secondaryMode, secondaryModeWarnings := getSecondaryModeStr(tryAllPrimariesBeforeSecondary, atsMajorVersion, dsName) warnings = append(warnings, secondaryModeWarnings...) // secondaryParents += secondaryModeStr // } else { diff --git a/lib/go-atscfg/parentdotconfig_test.go b/lib/go-atscfg/parentdotconfig_test.go index c5cb2a9cd4..9e09790a43 100644 --- a/lib/go-atscfg/parentdotconfig_test.go +++ b/lib/go-atscfg/parentdotconfig_test.go @@ -4437,6 +4437,251 @@ func TestMakeParentDotConfigFirstInnerLastTopology(t *testing.T) { } } +func TestMakeParentDotConfigOptVersion(t *testing.T) { + ds1 := makeParentDS() + ds1.ID = util.IntPtr(43) + ds1Type := tc.DSTypeDNS + ds1.Type = &ds1Type + ds1.QStringIgnore = util.IntPtr(int(tc.QStringIgnoreDrop)) + ds1.OrgServerFQDN = util.StrPtr("http://ds1.example.net") + ds1.Topology = util.StrPtr("t0") + ds1.ProfileName = util.StrPtr("ds1Profile") + ds1.ProfileID = util.IntPtr(994) + ds1.MultiSiteOrigin = util.BoolPtr(true) + + dses := []DeliveryService{*ds1} + + parentConfigParams := []tc.Parameter{ + tc.Parameter{ + Name: ParentConfigParamQStringHandling, + ConfigFile: "parent.config", + Value: "myQStringHandlingParam", + Profiles: []byte(`["serverprofile"]`), + }, + tc.Parameter{ + Name: ParentConfigRetryKeysDefault.Algorithm, + ConfigFile: "parent.config", + Value: tc.AlgorithmConsistentHash, + Profiles: []byte(`["serverprofile"]`), + }, + tc.Parameter{ + Name: ParentConfigParamQString, + ConfigFile: "parent.config", + Value: "myQstringParam", + Profiles: []byte(`["serverprofile"]`), + }, + tc.Parameter{ + Name: ParentConfigRetryKeysDefault.Algorithm, + ConfigFile: "parent.config", + Value: "consistent_hash", + Profiles: []byte(`["ds1Profile"]`), + }, + tc.Parameter{ + Name: ParentConfigRetryKeysDefault.ParentRetry, + ConfigFile: "parent.config", + Value: "both", + Profiles: []byte(`["ds1Profile"]`), + }, + tc.Parameter{ + Name: ParentConfigRetryKeysDefault.MaxSimpleRetries, + ConfigFile: "parent.config", + Value: "14", + Profiles: []byte(`["ds1Profile"]`), + }, + tc.Parameter{ + Name: ParentConfigRetryKeysDefault.MaxUnavailableRetries, + ConfigFile: "parent.config", + Value: "9", + Profiles: []byte(`["ds1Profile"]`), + }, + tc.Parameter{ + Name: ParentConfigRetryKeysDefault.UnavailableRetryResponses, + ConfigFile: "parent.config", + Value: `"400,503"`, + Profiles: []byte(`["ds1Profile"]`), + }, + } + + server := makeTestParentServer() + server.Cachegroup = util.StrPtr("edgeCG") + server.CachegroupID = util.IntPtr(400) + + origin0 := makeTestParentServer() + origin0.Cachegroup = util.StrPtr("originCG") + origin0.CachegroupID = util.IntPtr(500) + origin0.HostName = util.StrPtr("myorigin0") + origin0.ID = util.IntPtr(45) + setIP(origin0, "192.168.2.2") + origin0.Type = tc.OriginTypeName + origin0.TypeID = util.IntPtr(991) + + origin1 := makeTestParentServer() + origin1.Cachegroup = util.StrPtr("originCG") + origin1.CachegroupID = util.IntPtr(500) + origin1.HostName = util.StrPtr("myorigin1") + origin1.ID = util.IntPtr(46) + setIP(origin1, "192.168.2.3") + origin1.Type = tc.OriginTypeName + origin1.TypeID = util.IntPtr(991) + + servers := []Server{*server, *origin0, *origin1} + + topologies := []tc.Topology{ + tc.Topology{ + Name: "t0", + Nodes: []tc.TopologyNode{ + tc.TopologyNode{ + Cachegroup: "edgeCG", + Parents: []int{1}, + }, + tc.TopologyNode{ + Cachegroup: "originCG", + }, + }, + }, + } + + serverCapabilities := map[int]map[ServerCapability]struct{}{} + dsRequiredCapabilities := map[int]map[ServerCapability]struct{}{} + + eCG := &tc.CacheGroupNullable{} + eCG.Name = server.Cachegroup + eCG.ID = server.CachegroupID + eCG.ParentName = origin0.Cachegroup + eCG.ParentCachegroupID = origin0.CachegroupID + eCGType := tc.CacheGroupEdgeTypeName + eCG.Type = &eCGType + + oCG := &tc.CacheGroupNullable{} + oCG.Name = origin0.Cachegroup + oCG.ID = origin0.CachegroupID + oCGType := tc.CacheGroupOriginTypeName + oCG.Type = &oCGType + + cgs := []tc.CacheGroupNullable{*eCG, *oCG} + + dss := []DeliveryServiceServer{ + DeliveryServiceServer{ + Server: *origin0.ID, + DeliveryService: *ds1.ID, + }, + } + cdn := &tc.CDN{ + DomainName: "cdndomain.example", + Name: "my-cdn-name", + } + + // unavailable_server_retry_responses is not available as a feature in ATS 5, but is in ATS 9. + + t.Run("Package Parameter 9 with no Opt ATSVersion has ATS 9 feature", func(t *testing.T) { + serverParams := []tc.Parameter{ + tc.Parameter{ + Name: "trafficserver", + ConfigFile: "package", + Value: "9", + Profiles: []byte(`["global"]`), + }, + } + + opt := &ParentConfigOpts{AddComments: false, HdrComment: "myHeaderComment"} + + cfg, err := MakeParentDotConfig(dses, server, servers, topologies, serverParams, parentConfigParams, serverCapabilities, dsRequiredCapabilities, cgs, dss, cdn, opt) + if err != nil { + t.Fatal(err) + } + txt := cfg.Text + + testComment(t, txt, opt.HdrComment) + + if !strings.Contains(txt, `unavailable_server_retry_responses="400,503"`) { + t.Errorf(`expected Package Parameter ATS 9 with no Opt ATS Version to have unavailable_server_retry_responses feature', actual: '%v'`, txt) + } + }) + + t.Run("Package Parameter 5 with no Opt ATSVersion does not have the feature it shouldn't", func(t *testing.T) { + serverParams := []tc.Parameter{ + tc.Parameter{ + Name: "trafficserver", + ConfigFile: "package", + Value: "5", + Profiles: []byte(`["global"]`), + }, + } + + opt := &ParentConfigOpts{AddComments: false, HdrComment: "myHeaderComment"} + + cfg, err := MakeParentDotConfig(dses, server, servers, topologies, serverParams, parentConfigParams, serverCapabilities, dsRequiredCapabilities, cgs, dss, cdn, opt) + if err != nil { + t.Fatal(err) + } + txt := cfg.Text + + testComment(t, txt, opt.HdrComment) + + if strings.Contains(txt, `unavailable_server_retry_responses`) { + t.Errorf(`expected Package Parameter ATS 5 with no Opt ATS Version to not have unavailable_server_retry_responses feature', actual: '%v'`, txt) + } + }) + + t.Run("Package Parameter 5 with Opt ATSVersion 9 uses Opt not Param.", func(t *testing.T) { + serverParams := []tc.Parameter{ + tc.Parameter{ + Name: "trafficserver", + ConfigFile: "package", + Value: "5", + Profiles: []byte(`["global"]`), + }, + } + + opt := &ParentConfigOpts{ + AddComments: false, + HdrComment: "myHeaderComment", + ATSMajorVersion: 9, + } + + cfg, err := MakeParentDotConfig(dses, server, servers, topologies, serverParams, parentConfigParams, serverCapabilities, dsRequiredCapabilities, cgs, dss, cdn, opt) + if err != nil { + t.Fatal(err) + } + txt := cfg.Text + + testComment(t, txt, opt.HdrComment) + + if !strings.Contains(txt, `unavailable_server_retry_responses`) { + t.Errorf(`expected Package Parameter ATS 5 with Opt ATS Version 9 to use Opt not Parameter with ATS 9 unavailable_server_retry_responses feature', actual: '%v'`, txt) + } + }) + + t.Run("Package Parameter 9 with Opt ATSVersion 5 uses Opt not Param.", func(t *testing.T) { + serverParams := []tc.Parameter{ + tc.Parameter{ + Name: "trafficserver", + ConfigFile: "package", + Value: "9", + Profiles: []byte(`["global"]`), + }, + } + + opt := &ParentConfigOpts{ + AddComments: false, + HdrComment: "myHeaderComment", + ATSMajorVersion: 5, + } + + cfg, err := MakeParentDotConfig(dses, server, servers, topologies, serverParams, parentConfigParams, serverCapabilities, dsRequiredCapabilities, cgs, dss, cdn, opt) + if err != nil { + t.Fatal(err) + } + txt := cfg.Text + + testComment(t, txt, opt.HdrComment) + + if strings.Contains(txt, `unavailable_server_retry_responses`) { + t.Errorf(`expected Package Parameter ATS 9 with Opt ATS Version 5 to use Opt not Parameter with no ATS 9 unavailable_server_retry_responses feature', actual: '%v'`, txt) + } + }) +} + func TestMakeParentDotConfigOriginIP(t *testing.T) { hdr := &ParentConfigOpts{AddComments: false, HdrComment: "myHeaderComment"} diff --git a/lib/go-atscfg/remapdotconfig.go b/lib/go-atscfg/remapdotconfig.go index 4c82418894..31b02dc65d 100644 --- a/lib/go-atscfg/remapdotconfig.go +++ b/lib/go-atscfg/remapdotconfig.go @@ -56,6 +56,14 @@ type RemapDotConfigOpts struct { // UseCoreStrategies is whether to use the ATS core strategies, rather than the parent_select plugin. // This has no effect if UseStrategies is false. UseStrategiesCore bool + + // ATSMajorVersion is the integral major version of Apache Traffic server, + // used to generate the proper config for the proper version. + // + // If omitted or 0, the major version will be read from the Server's Profile Parameter config file 'package' name 'trafficserver'. If no such Parameter exists, the ATS version will default to 5. + // This was the old Traffic Control behavior, before the version was specifiable externally. + // + ATSMajorVersion uint } func MakeRemapDotConfig( @@ -104,8 +112,8 @@ func MakeRemapDotConfig( warnings = append(warnings, "making Delivery Service Cache Key Params, cache key will be missing! : "+err.Error()) } - atsMajorVersion, verWarns := getATSMajorVersion(serverParams) - warnings = append(warnings, verWarns...) + atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, serverParams, &warnings) + serverPackageParamData, paramWarns := makeServerPackageParamData(server, serverParams) warnings = append(warnings, paramWarns...) cacheGroups, err := makeCGMap(cacheGroupArr) @@ -246,7 +254,7 @@ func lastPrePostRemapLinesFor(dsConfigParamsMap map[string][]tc.Parameter, dsid // getServerConfigRemapDotConfigForMid returns the remap lines, any warnings, and any error. func getServerConfigRemapDotConfigForMid( - atsMajorVersion int, + atsMajorVersion uint, profilesConfigParams map[int][]tc.Parameter, dses []DeliveryService, dsRegexes map[tc.DeliveryServiceName][]tc.DeliveryServiceRegex, @@ -389,7 +397,7 @@ func getServerConfigRemapDotConfigForEdge( serverPackageParamData map[string]string, // map[paramName]paramVal for this server, config file 'package' dses []DeliveryService, dsRegexes map[tc.DeliveryServiceName][]tc.DeliveryServiceRegex, - atsMajorVersion int, + atsMajorVersion uint, header string, server *Server, nameTopologies map[TopologyName]tc.Topology, @@ -497,7 +505,7 @@ type RemapLines struct { // The cacheKeyConfigParams map may be nil, if this ds profile had no cache key config params. // Returns the remap line, any warnings, and any error. func buildEdgeRemapLine( - atsMajorVersion int, + atsMajorVersion uint, server *Server, pData map[string]string, text string, @@ -739,7 +747,7 @@ func midHeaderRewriteConfigFileName(dsName string) string { } // getQStringIgnoreRemap returns the remap, whether cachekey was added. -func getQStringIgnoreRemap(atsMajorVersion int) string { +func getQStringIgnoreRemap(atsMajorVersion uint) string { if atsMajorVersion < 7 { log.Errorf("Unsupport version of ats found %v", atsMajorVersion) return "" diff --git a/lib/go-atscfg/strategiesdotconfig.go b/lib/go-atscfg/strategiesdotconfig.go index b84b113bdf..c74b26bffa 100644 --- a/lib/go-atscfg/strategiesdotconfig.go +++ b/lib/go-atscfg/strategiesdotconfig.go @@ -41,6 +41,14 @@ type StrategiesYAMLOpts struct { // This should be the text desired, without comment syntax (like # or //). The file's comment syntax will be added. // To omit the header comment, pass the empty string. HdrComment string + + // ATSMajorVersion is the integral major version of Apache Traffic server, + // used to generate the proper config for the proper version. + // + // If omitted or 0, the major version will be read from the Server's Profile Parameter config file 'package' name 'trafficserver'. If no such Parameter exists, the ATS version will default to 5. + // This was the old Traffic Control behavior, before the version was specifiable externally. + // + ATSMajorVersion uint } func MakeStrategiesDotYAML( @@ -57,10 +65,14 @@ func MakeStrategiesDotYAML( cdn *tc.CDN, opt *StrategiesYAMLOpts, ) (Cfg, error) { + warnings := []string{} if opt == nil { opt = &StrategiesYAMLOpts{} } - parentAbstraction, warnings, err := makeParentDotConfigData( + + atsMajorVersion := getATSMajorVersion(opt.ATSMajorVersion, tcServerParams, &warnings) + + parentAbstraction, dataWarns, err := makeParentDotConfigData( dses, server, servers, @@ -73,18 +85,18 @@ func MakeStrategiesDotYAML( dss, cdn, &ParentConfigOpts{ - AddComments: opt.VerboseComments, - HdrComment: opt.HdrComment, + AddComments: opt.VerboseComments, + HdrComment: opt.HdrComment, + ATSMajorVersion: opt.ATSMajorVersion, }, // TODO change makeParentDotConfigData to its own opt? + atsMajorVersion, ) + warnings = append(warnings, dataWarns...) if err != nil { return Cfg{}, makeErr(warnings, err.Error()) } - atsMajorVer, verWarns := getATSMajorVersion(tcServerParams) - warnings = append(warnings, verWarns...) - - text, paWarns, err := parentAbstractionToStrategiesDotYaml(parentAbstraction, opt, atsMajorVer) + text, paWarns, err := parentAbstractionToStrategiesDotYaml(parentAbstraction, opt, atsMajorVersion) warnings = append(warnings, paWarns...) if err != nil { return Cfg{}, makeErr(warnings, err.Error()) @@ -107,7 +119,7 @@ func MakeStrategiesDotYAML( const YAMLDocumentStart = "---" const YAMLDocumentEnd = "..." -func parentAbstractionToStrategiesDotYaml(pa *ParentAbstraction, opt *StrategiesYAMLOpts, atsMajorVersion int) (string, []string, error) { +func parentAbstractionToStrategiesDotYaml(pa *ParentAbstraction, opt *StrategiesYAMLOpts, atsMajorVersion uint) (string, []string, error) { warnings := []string{} txt := YAMLDocumentStart + getStrategyHostsSection(pa) +