Merge branch 'master' into tpv2-redirect

CHANGELOG.md

@@ -45,6 +45,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - [#7619](https://github.com/apache/trafficcontrol/pull/7619) Traffic Ops* added optional field `oauth_user_attribute` for OAuth login credentials
 - [#7641](https://github.com/apache/trafficcontrol/pull/7641) *Traffic Router* Added further optimization to TR's algorithm of figuring out the zone for an incoming request.
 - [#7646](https://github.com/apache/trafficcontrol/pull/7646) *Traffic Portal* Add the ability to delete a cert.
+- [#7652](https://github.com/apache/trafficcontrol/pull/7652) *t3c* added rpmdb checks and use package data from t3c-apply-metadata.json if rpmdb is corrupt
 
 ### Changed
 - [#7584](https://github.com/apache/trafficcontrol/pull/7584) *Documentation* Upgrade Traffic Control Sphinx documentation Makefile OS intelligent.

cache-config/t3c-apply/config/config.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -81,6 +82,7 @@ type Cfg struct {
 	SvcManagement       SvcManagement
 	Retries             int
 	ReverseProxyDisable bool
+	RpmDBOk             bool
 	SkipOSCheck         bool
 	UseStrategies       t3cutil.UseStrategiesFlag
 	TOInsecure          bool
@@ -188,6 +190,29 @@ func directoryExists(dir string) (bool, os.FileInfo) {
 	return info.IsDir(), info
 }
 
+const rpmDir = "/var/lib/rpm"
+
+// verifies the rpm database files. if there is any database corruption
+// it will return false
+func verifyRpmDB() bool {
+	exclude := regexp.MustCompile(`(^\.|^__)`)
+	dbFiles, err := os.ReadDir(rpmDir)
+	if err != nil {
+		return false
+	}
+	for _, file := range dbFiles {
+		if exclude.Match([]byte(file.Name())) {
+			continue
+		}
+		cmd := exec.Command("/usr/lib/rpm/rpmdb_verify", rpmDir+"/"+file.Name())
+		err := cmd.Run()
+		if err != nil || cmd.ProcessState.ExitCode() > 0 {
+			return false
+		}
+	}
+	return true
+}
+
 // derives the ATS Installation directory from
 // the rpm config file list.
 func GetTSPackageHome() string {
@@ -322,10 +347,11 @@ If any of the related flags are also set, they override the mode's default behav
 	// so we want to log what flags the mode set here, to aid debugging.
 	// But we can't do that until the loggers are initialized.
 	modeLogStrs := []string{}
+	fatalLogStrs := []string{}
 	if getopt.IsSet(runModeFlagName) {
 		runMode := t3cutil.StrToMode(*runModePtr)
 		if runMode == t3cutil.ModeInvalid {
-			return Cfg{}, errors.New(*runModePtr + " is an invalid mode.")
+			fatalLogStrs = append(fatalLogStrs, *runModePtr+" is an invalid mode.")
 		}
 		modeLogStrs = append(modeLogStrs, "t3c-apply is running in "+runMode.String()+" mode")
 		switch runMode {
@@ -411,7 +437,7 @@ If any of the related flags are also set, they override the mode's default behav
 	}
 
 	if *verbosePtr > 2 {
-		return Cfg{}, errors.New("Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
+		fatalLogStrs = append(fatalLogStrs, "Too many verbose options. The maximum log verbosity level is 2 (-vv or --verbose=2) for errors (0), warnings (1), and info (2)")
 	}
 
 	var cacheHostName string
@@ -420,7 +446,7 @@ If any of the related flags are also set, they override the mode's default behav
 	} else {
 		cacheHostName, err = os.Hostname()
 		if err != nil {
-			return Cfg{}, errors.New("Could not get the hostname from the O.S., please supply a hostname: " + err.Error())
+			fatalLogStrs = append(fatalLogStrs, "Could not get the hostname from the O.S., please supply a hostname: "+err.Error())
 		}
 		// strings.Split always returns a slice with at least 1 element, so we don't need a len check
 		cacheHostName = strings.Split(cacheHostName, ".")[0]
@@ -429,7 +455,7 @@ If any of the related flags are also set, they override the mode's default behav
 	useGit := StrToUseGitFlag(*useGitStr)
 
 	if useGit == UseGitInvalid {
-		return Cfg{}, errors.New("Invalid git flag '" + *useGitStr + "'. Valid options are yes, no, auto.")
+		fatalLogStrs = append(fatalLogStrs, "Invalid git flag '"+*useGitStr+"'. Valid options are yes, no, auto.")
 	}
 
 	retries := *retriesPtr
@@ -471,6 +497,17 @@ If any of the related flags are also set, they override the mode's default behav
 		os.Setenv("TO_PASS", toPass)
 	}
 
+	rpmDBisOk := verifyRpmDB()
+
+	if *installPackagesPtr && !rpmDBisOk {
+		if t3cutil.StrToMode(*runModePtr) == t3cutil.ModeBadAss {
+			fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue in badass mode")
+		} else {
+			fatalLogStrs = append(fatalLogStrs, "RPM database check failed unable to install packages cannot continue")
+		}
+	}
+
+	toInfoLog = append(toInfoLog, fmt.Sprintf("rpm database is ok: %t", rpmDBisOk))
 	// set TSHome
 	var tsHome = ""
 	if *tsHomePtr != "" {
@@ -481,13 +518,13 @@ If any of the related flags are also set, they override the mode's default behav
 		tsHome = os.Getenv("TS_HOME") // check for the environment variable.
 		if tsHome != "" {
 			toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from TS_HOME environment variable '%s'\n", TSHome))
-		} else { // finally check using the config file listing from the rpm package.
+		} else if rpmDBisOk { // check using the config file listing from the rpm package if rpmdb is ok.
 			tsHome = GetTSPackageHome()
 			if tsHome != "" {
 				toInfoLog = append(toInfoLog, fmt.Sprintf("set TSHome from the RPM config file  list '%s'\n", TSHome))
-			} else {
-				toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
 			}
+		} else if tsHome == "" {
+			toInfoLog = append(toInfoLog, fmt.Sprintf("no override for TSHome was found, using the configured default: '%s'\n", TSHome))
 		}
 	}
 
@@ -503,23 +540,23 @@ If any of the related flags are also set, they override the mode's default behav
 	if *useLocalATSVersionPtr {
 		atsVersionStr, err = GetATSVersionStr(tsHome)
 		if err != nil {
-			return Cfg{}, errors.New("getting local ATS version: " + err.Error())
+			fatalLogStrs = append(fatalLogStrs, "getting local ATS version: "+err.Error())
 		}
 	}
 	toInfoLog = append(toInfoLog, fmt.Sprintf("ATSVersionStr: '%s'\n", atsVersionStr))
 
 	usageStr := "basic usage: t3c-apply --traffic-ops-url=myurl --traffic-ops-user=myuser --traffic-ops-password=mypass --cache-host-name=my-cache"
 	if strings.TrimSpace(toURL) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-url or TO_URL environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-url or TO_URL environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(toUser) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-user or TO_USER environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-user or TO_USER environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(toPass) == "" {
-		return Cfg{}, errors.New("Missing required argument --traffic-ops-password or TO_PASS environment variable. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --traffic-ops-password or TO_PASS environment variable. "+usageStr)
 	}
 	if strings.TrimSpace(cacheHostName) == "" {
-		return Cfg{}, errors.New("Missing required argument --cache-host-name. " + usageStr)
+		fatalLogStrs = append(fatalLogStrs, "Missing required argument --cache-host-name. "+usageStr)
 	}
 
 	toURLParsed, err := url.Parse(toURL)
@@ -540,6 +577,7 @@ If any of the related flags are also set, they override the mode's default behav
 		CacheHostName:               cacheHostName,
 		SvcManagement:               svcManagement,
 		Retries:                     retries,
+		RpmDBOk:                     rpmDBisOk,
 		ReverseProxyDisable:         reverseProxyDisable,
 		SkipOSCheck:                 skipOsCheck,
 		UseStrategies:               useStrategies,
@@ -580,6 +618,13 @@ If any of the related flags are also set, they override the mode's default behav
 		return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")
 	}
 
+	if len(fatalLogStrs) > 0 {
+		for _, str := range fatalLogStrs {
+			str = strings.TrimSpace(str)
+			log.Errorln(str)
+		}
+		return Cfg{}, errors.New("fatal error has occurred")
+	}
 	for _, str := range modeLogStrs {
 		str = strings.TrimSpace(str)
 		if str == "" {

cache-config/t3c-apply/t3c-apply.go

@@ -22,7 +22,6 @@ package main
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -94,8 +93,8 @@ func Main() int {
 	var lock util.FileLock
 	cfg, err := config.GetCfg(Version, GitRevision)
 	if err != nil {
-		fmt.Println(err)
-		fmt.Println(FailureExitMsg)
+		log.Infoln(err)
+		log.Errorln(FailureExitMsg)
 		return ExitCodeConfigError
 	} else if cfg == (config.Cfg{}) { // user used the --help option
 		return ExitCodeSuccess
@@ -261,7 +260,7 @@ func Main() int {
 		// make sure we got the data necessary to check packages
 		log.Infoln("======== Didn't get all files, no package processing needed or possible ========")
 		metaData.InstalledPackages = oldMetaData.InstalledPackages
-	} else {
+	} else if cfg.RpmDBOk {
 		log.Infoln("======== Start processing packages  ========")
 		err = trops.ProcessPackages()
 		if err != nil {
@@ -276,6 +275,9 @@ func Main() int {
 			log.Errorf("Error verifying system services: %s\n", err.Error())
 			return GitCommitAndExit(ExitCodeServicesError, FailureExitMsg, cfg, metaData, oldMetaData)
 		}
+	} else {
+		log.Warnln("======== RPM DB checks failed, package processing not possible, using installed packages from  metadata if available========")
+		trops.ProcessPackagesWithMetaData(oldMetaData.InstalledPackages)
 	}
 
 	log.Debugf("Preparing to fetch the config files for %s, files: %s, syncdsUpdate: %s\n", cfg.CacheHostName, cfg.Files, syncdsUpdate)

cache-config/t3c-apply/torequest/torequest.go

@@ -587,10 +587,14 @@ func (r *TrafficOpsReq) CheckSystemServices() error {
 func (r *TrafficOpsReq) IsPackageInstalled(name string) bool {
 	for k, v := range r.Pkgs {
 		if strings.HasPrefix(k, name) {
+			log.Infof("Found in cache for '%s'", k)
 			return v
 		}
 	}
-
+	if !r.Cfg.RpmDBOk {
+		log.Warnf("RPM DB is corrupted cannot run IsPackageInstalled for '%s' and package metadata is unavailable", name)
+		return false
+	}
 	log.Infof("IsPackageInstalled '%v' not found in cache, querying rpm", name)
 	pkgArr, err := util.PackageInfo("pkg-query", name)
 	if err != nil {
@@ -1030,6 +1034,47 @@ func (r *TrafficOpsReq) ProcessPackages() error {
 	return nil
 }
 
+func pkgMetaDataToMap(pmd []string) map[string]bool {
+	pkgMap := map[string]bool{}
+	for _, pkg := range pmd {
+		pkgMap[pkg] = true
+	}
+	return pkgMap
+}
+
+func pkgMatch(pkgMetaData []string, pk string) bool {
+	for _, pkg := range pkgMetaData {
+		if strings.Contains(pk, pkg) {
+			return true
+		}
+	}
+	return false
+
+}
+
+// ProcessPackagesWithMetaData will attempt to get installed package data from
+// t3c-apply-metadata.json and log the results.
+func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) error {
+	pkgs, err := getPackages(r.Cfg)
+	pkgMdataMap := pkgMetaDataToMap(packageMetaData)
+	if err != nil {
+		return fmt.Errorf("getting packages: %w", err)
+	}
+	for _, pkg := range pkgs {
+		fullPackage := pkg.Name + "-" + pkg.Version
+		if pkgMdataMap[fullPackage] {
+			log.Infof("package %s is assumed to be installed according to metadata file", fullPackage)
+			r.Pkgs[fullPackage] = true
+		} else if pkgMatch(packageMetaData, pkg.Name) {
+			log.Infof("package %s is assumed to be installed according to metadata, but doesn't match traffic ops pkg", fullPackage)
+			r.Pkgs[fullPackage] = true
+		} else {
+			log.Infof("package %s does not appear to be installed.", pkg.Name+"-"+pkg.Version)
+		}
+	}
+	return nil
+}
+
 func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
 	updateStatus, err := r.CheckRevalidateState(true)
 	if err != nil {

infrastructure/cdn-in-a-box/health/tc-health-client-ReadMe.md

@@ -0,0 +1,38 @@
+<!--
+	Licensed to the Apache Software Foundation (ASF) under one
+	or more contributor license agreements.  See the NOTICE file
+	distributed with this work for additional information
+	regarding copyright ownership.  The ASF licenses this file
+	to you under the Apache License, Version 2.0 (the
+	"License"); you may not use this file except in compliance
+	with the License.  You may obtain a copy of the License at
+
+	  http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing,
+	software distributed under the License is distributed on an
+	"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+	KIND, either express or implied.  See the License for the
+	specific language governing permissions and limitations
+	under the License.
+-->
+
+# CDN-in-a-Box Health Client Testing
+
+## Building and Running
+
+Build and run cdn-in-a-box `docker-compose -f docker-compose.yml -f docker-compose.expose-ports.yml up`, once up and running, using docker desktop, navigate to terminal tab of an edge or mid. cd into `/var/log/trafficcontrol` and run `tail -f tc-health-client.log`. Click on the `Open in external terminal` on upper right side and cd into `/usr/bin` and run `./tc-health-client`. Wait for the dispersion time to pass and then logs will start in the window where the tail command was ran. After that you may interact with it via Traffic Portal.
+
+## Config files for Testing Only
+
+For testing only the `tc-health-client.json` are the settings used to run it locally and can be changed. If changed `purge` all containers and run `docker-compose -f docker-compose.yml -f docker-compose.expose-ports.yml up` in the `infrastructure/cdn-in-a-box/` folder. Same applies if the `tc-health-client.service` and `to-creds` files are changed. The `tc-health-client.service` is set for `Debug` mode with `vvv` which is different from Production which is `vv`.
+
+## Rebuilding the tc-health-client only 
+
+Delete the `trafficcontrol-health-client-[version].rpm` from the `\dist` folder and from `/trafficcontrol/infrastructure/cdn-in-a-box/health` then cd into `/trafficcontrol` and run `./pkg -v -8 -b tc-health-client_build` this builds the RPM to be used with docker or `./pkg -v -8 tc-health-client_build` to build x86_64. Then copy the rpm from `/dist` into `/trafficcontrol/infrastructure/cdn-in-a-box/health` and rename it to `trafficcontrol-health-client.rpm` by removing the version. Build and run with `docker-compose -f docker-compose.yml -f docker-compose`.
+
+## Example Testing Commands
+
+Cd into `/opt/trafficserver/bin/` and run `./traffic_ctl host down --reason active mid-01.infra.ciab.test` or `./traffic_ctl host status mid-01.infra.ciab.test` update it as needed for other servers or reason codes. 
+
+At the `/opt` level of a running containder for either edge or mid run `curl -vL http://trafficmonitor.infra.ciab.test:80` to test traffic_monitor

infrastructure/cdn-in-a-box/traffic_router/Dockerfile

@@ -52,7 +52,7 @@ RUN dnf -y install epel-release && \
 
         perl-JSON perl-WWW-Curl which make autoconf automake gcc gcc-c++ apr apr-devel \
         openssl openssl-devel bind-utils net-tools perl-JSON-PP gettext \
-        java-11-openjdk-headless java-11-openjdk-devel tomcat-native && \
+        java-11-openjdk-headless tzdata-java java-11-openjdk-devel tomcat-native && \
     dnf -y clean all && \
     ln -sfv $(realpath /usr/lib/jvm/java-11) /opt/java
 

infrastructure/docker/build/Dockerfile-traffic_router

@@ -46,6 +46,8 @@ RUN	yum -y install \
 		which  \
 		curl \
 		java-11-openjdk \
+		# necessary in case tzdata-java is not already installed as a dependency of java-11-openjdk-headless
+		tzdata-java \
 		java-11-openjdk-devel && \
 	yum -y clean all
 

tc-health-client/tmagent/markdownservice.go

@@ -309,7 +309,7 @@ func getParentFQDNs(pi *ParentInfo, tmh *TrafficMonitorHealth, l4h *ParentHealth
 //
 // This is a safety mechanism: if for any reason most or all parents are marked down, something
 // is seriously wrong, possibly with the health code itself, and therefore don't mark any parents down,
-const HealthSafetyRatio = 0.3 // TODO make configurable? // 0.3
+const HealthSafetyRatio = 0.3 // TODO make configurable?
 
 func doMarkdown(pi *ParentInfo) {
 	cfg := pi.Cfg.Get()
@@ -360,7 +360,7 @@ func doMarkdown(pi *ParentInfo) {
 		if oldAvailable != newAvailable {
 			// do not mark down if the configuration disables mark downs.
 			if !cfg.EnableActiveMarkdowns && !newAvailable {
-				log.Infof("markdown monitored_host=%v host_status=%v event=TM reports host is not available", fqdn, pv.Status())
+				log.Infof("markdown monitored_host=%v host_status=%v event=\"TM reports host is not available\"", fqdn, pv.Status())
 			} else {
 				if newParentStatus, err := markParent(cfg, parentStatus, isAvailable.Status, newAvailable); err != nil {
 					log.Errorln(err.Error())
@@ -409,7 +409,7 @@ func markParent(cfg *config.Cfg, pv ParentStatus, cacheStatus string, available
 	if !available { // unavailable
 		unavailablePollCount += 1
 		if unavailablePollCount < cfg.UnavailablePollThreshold {
-			log.Infof("markdown monitored_host=%v host_status=%v event=TM indicates host is unavailable but the UnavailablePollThreshold has not been reached", hostName, hostStatus)
+			log.Infof("markdown monitored_host=%v host_status=UNAVAILABLE event=\"TM indicates host is unavailable but the UnavailablePollThreshold has not been reached\"", hostName)
 			hostAvailable = true
 		} else {
 			// marking the host down
@@ -422,13 +422,13 @@ func markParent(cfg *config.Cfg, pv ParentStatus, cacheStatus string, available
 			// reset the poll counts
 			markUpPollCount = 0
 			unavailablePollCount = 0
-			log.Infof("marked monitored_host=%v host_status=%v event=%v\n", hostName, hostStatus, cacheStatus)
+			log.Infof("marked monitored_host=%v host_status=%v event=\"%v\"\n", hostName, hostStatus, cacheStatus)
 		}
 	} else { // available
 		// marking the host up
 		markUpPollCount += 1
 		if markUpPollCount < cfg.MarkUpPollThreshold {
-			log.Infof("TM indicates %s is available but the MarkUpPollThreshold has not been reached", hostName)
+			log.Infof("monitored_host=%v event=\"TM indicates host is available but the MarkUpPollThreshold has not been reached\"", hostName)
 			hostAvailable = false
 		} else {
 			if err := execTrafficCtl(pv.Fqdn, true, cfg.ReasonCode, cfg.TrafficServerBinDir); err != nil {
@@ -439,7 +439,7 @@ func markParent(cfg *config.Cfg, pv ParentStatus, cacheStatus string, available
 			// reset the poll counts
 			unavailablePollCount = 0
 			markUpPollCount = 0
-			log.Infof("markdown monitored_host=%v host_status=%v event=%v\n", hostName, hostStatus, cacheStatus)
+			log.Infof("markdown monitored_host=%v host_status=%v event=\"%v\"\n", hostName, hostStatus, cacheStatus)
 		}
 	}