From 0c5e0ffefe25ec4b03c891ed2526eda227a306ed Mon Sep 17 00:00:00 2001 From: Ahmed Elsabbahy Date: Sun, 24 Jul 2016 15:45:29 -0400 Subject: [PATCH] Add concept of skipped tests * Skip optional tests for non existing system resources * Skipped tests do not count as a failure on their own, but are almost always accompanied with a failing test, the only exception is when the exists test is made optional through matchers (see #107): ```yaml exists: or: - true - false ``` * Undefined required attributes will now result in an error --- outputs/documentation.go | 46 ++++++++++-------------- outputs/junit.go | 28 +++++++-------- outputs/nagios.go | 12 ++++--- outputs/outputs.go | 75 ++++++++++++++++++++++++++++++--------- outputs/rspecish.go | 43 +++++++++------------- outputs/tap.go | 9 +++-- resource/addr.go | 15 ++++---- resource/command.go | 10 +++--- resource/dns.go | 13 +++---- resource/file.go | 31 +++++++--------- resource/gomega.go | 3 ++ resource/group.go | 13 +++---- resource/interface.go | 12 ++++--- resource/kernel_param.go | 13 ++++--- resource/mount.go | 16 +++++---- resource/package.go | 13 +++---- resource/port.go | 13 +++---- resource/process.go | 13 ++++--- resource/resource.go | 7 ++++ resource/service.go | 17 +++++---- resource/user.go | 21 +++++------ resource/validate.go | 71 ++++++++++++++++++++++++++++++------ resource/validate_test.go | 38 ++++++++++++++++---- 23 files changed, 322 insertions(+), 210 deletions(-) diff --git a/outputs/documentation.go b/outputs/documentation.go index a60560a58..7000de917 100644 --- a/outputs/documentation.go +++ b/outputs/documentation.go @@ -5,59 +5,49 @@ import ( "time" "github.com/aelsabbahy/goss/resource" - "github.com/fatih/color" ) type Documentation struct{} func (r Documentation) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) { testCount := 0 - var failed [][]resource.TestResult + var failedOrSkipped [][]resource.TestResult + var skipped, failed int for resultGroup := range results { - failedGroup := []resource.TestResult{} + failedOrSkippedGroup := []resource.TestResult{} first := resultGroup[0] header := header(first) if header != "" { fmt.Print(header) } for _, testResult := range resultGroup { - if testResult.Successful { + switch testResult.Result { + case resource.SUCCESS: fmt.Println(humanizeResult(testResult)) testCount++ - } else { + case resource.SKIP: fmt.Println(humanizeResult(testResult)) - failedGroup = append(failedGroup, testResult) - testCount++ + failedOrSkippedGroup = append(failedOrSkippedGroup, testResult) + skipped++ + case resource.FAIL: + fmt.Println(humanizeResult(testResult)) + failedOrSkippedGroup = append(failedOrSkippedGroup, testResult) + failed++ } + testCount++ } - fmt.Println("") - if len(failedGroup) > 0 { - failed = append(failed, failedGroup) + if len(failedOrSkippedGroup) > 0 { + failedOrSkipped = append(failedOrSkipped, failedOrSkippedGroup) } } fmt.Print("\n\n") - if len(failed) > 0 { - fmt.Println("Failures:\n") - for _, failedGroup := range failed { - first := failedGroup[0] - header := header(first) - if header != "" { - fmt.Print(header) - } - for _, testResult := range failedGroup { - fmt.Println(humanizeResult(testResult)) - } - fmt.Print("\n") - } - } + fmt.Print(failedOrSkippedSummary(failedOrSkipped)) - fmt.Printf("Total Duration: %.3fs\n", time.Since(startTime).Seconds()) - if len(failed) > 0 { - color.Red("Count: %d, Failed: %d\n", testCount, len(failed)) + fmt.Print(summary(startTime, testCount, failed, skipped)) + if failed > 0 { return 1 } - color.Green("Count: %d, Failed: %d\n", testCount, len(failed)) return 0 } diff --git a/outputs/junit.go b/outputs/junit.go index 53a87d85c..85de5baba 100644 --- a/outputs/junit.go +++ b/outputs/junit.go @@ -11,8 +11,7 @@ import ( type JUnit struct{} func (r JUnit) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) { - testCount := 0 - failed := 0 + var testCount, failed, skipped int // ISO8601 timeformat location, _ := time.LoadLocation("Etc/UTC") @@ -30,22 +29,23 @@ func (r JUnit) Output(results <-chan []resource.TestResult, startTime time.Time) testResult.ResourceId + " " + testResult.Property + "\" " + "time=\"" + duration + "\">\n" - if testResult.Successful { - summary[testCount] = summary[testCount] + - "" + - humanizeResult2(testResult) + - "\n\n" - } else { - summary[testCount] = summary[testCount] + - "" + + if testResult.Result == resource.FAIL { + summary[testCount] += "" + humanizeResult2(testResult) + "\n" - summary[testCount] = summary[testCount] + - "" + + summary[testCount] += "" + humanizeResult2(testResult) + "\n\n" failed++ + } else { + if testResult.Result == resource.SKIP { + summary[testCount] += "" + skipped++ + } + summary[testCount] += "" + + humanizeResult2(testResult) + + "\n\n" } testCount++ } @@ -54,8 +54,8 @@ func (r JUnit) Output(results <-chan []resource.TestResult, startTime time.Time) duration := time.Since(startTime) fmt.Println("") fmt.Printf("\n", - testCount, failed, duration.Seconds(), timestamp) + "failures=\"%d\" skipped=\"%d\" time=\"%.3f\" timestamp=\"%s\">\n", + testCount, failed, skipped, duration.Seconds(), timestamp) for i := 0; i < testCount; i++ { fmt.Printf("%s", summary[i]) diff --git a/outputs/nagios.go b/outputs/nagios.go index 45fdf2e89..c486867d4 100644 --- a/outputs/nagios.go +++ b/outputs/nagios.go @@ -10,12 +10,14 @@ import ( type Nagios struct{} func (r Nagios) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) { - testCount := 0 - failed := 0 + var testCount, failed, skipped int for resultGroup := range results { for _, testResult := range resultGroup { - if !testResult.Successful { + switch testResult.Result { + case resource.FAIL: failed++ + case resource.SKIP: + skipped++ } testCount++ } @@ -23,10 +25,10 @@ func (r Nagios) Output(results <-chan []resource.TestResult, startTime time.Time duration := time.Since(startTime) if failed > 0 { - fmt.Printf("GOSS CRITICAL - Count: %d, Failed: %d, Duration: %.3fs\n", testCount, failed, duration.Seconds()) + fmt.Printf("GOSS CRITICAL - Count: %d, Failed: %d, Skipped: %d, Duration: %.3fs\n", testCount, failed, skipped, duration.Seconds()) return 2 } - fmt.Printf("GOSS OK - Count: %d, Failed: %d, Duration: %.3fs\n", testCount, failed, duration.Seconds()) + fmt.Printf("GOSS OK - Count: %d, Failed: %d, Skipped: %d, Duration: %.3fs\n", testCount, failed, skipped, duration.Seconds()) return 0 } diff --git a/outputs/outputs.go b/outputs/outputs.go index 4ba946396..8cd48c86b 100644 --- a/outputs/outputs.go +++ b/outputs/outputs.go @@ -18,19 +18,25 @@ type Outputer interface { var green = color.New(color.FgGreen).SprintfFunc() var red = color.New(color.FgRed).SprintfFunc() +var yellow = color.New(color.FgYellow).SprintfFunc() func humanizeResult(r resource.TestResult) string { if r.Err != nil { return red("%s: %s: Error: %s", r.ResourceId, r.Property, r.Err) } - if r.Successful { + switch r.Result { + case resource.SUCCESS: return green("%s: %s: %s: matches expectation: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected) - } else { + case resource.SKIP: + return yellow("%s: %s: %s: skipped", r.ResourceType, r.ResourceId, r.Property) + case resource.FAIL: if r.Human != "" { return red("%s: %s: %s:\n%s", r.ResourceType, r.ResourceId, r.Property, r.Human) } return humanizeResult2(r) + default: + panic(fmt.Sprintf("Unexpected Result Code: %v\n", r.Result)) } } @@ -39,27 +45,33 @@ func humanizeResult2(r resource.TestResult) string { return red("%s: %s: Error: %s", r.ResourceId, r.Property, r.Err) } - switch r.TestType { - case resource.Value: - if r.Successful { + switch r.Result { + case resource.SUCCESS: + switch r.TestType { + case resource.Value: return green("%s: %s: %s: matches expectation: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected) - } else { - return red("%s: %s: %s: doesn't match, expect: %s found: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected, r.Found) - } - case resource.Values: - if r.Successful { + case resource.Values: return green("%s: %s: %s: all expectations found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(r.Expected, ", ")) - } else { - return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) + case resource.Contains: + return green("%s: %s: %s: all expectations found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(r.Expected, ", ")) + default: + return red("Unexpected type %d", r.TestType) } - case resource.Contains: - if r.Successful { - return green("%s: %s: %s: all patterns found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(r.Expected, ", ")) - } else { + case resource.FAIL: + switch r.TestType { + case resource.Value: + return red("%s: %s: %s: doesn't match, expect: %s found: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected, r.Found) + case resource.Values: + return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) + case resource.Contains: return red("%s: %s: %s: patterns not found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", ")) + default: + return red("Unexpected type %d", r.TestType) } + case resource.SKIP: + return yellow("%s: %s: %s: skipped", r.ResourceType, r.ResourceId, r.Property) default: - return red("Unexpected type %d", r.TestType) + panic(fmt.Sprintf("Unexpected Result Code: %v\n", r.Result)) } } @@ -139,3 +151,32 @@ func header(t resource.TestResult) string { } return out } + +func summary(startTime time.Time, count, failed, skipped int) string { + var s string + s += fmt.Sprintf("Total Duration: %.3fs\n", time.Since(startTime).Seconds()) + f := green + if failed > 0 { + f = red + } + s += f("Count: %d, Failed: %d, Skipped: %d\n", count, failed, skipped) + return s +} +func failedOrSkippedSummary(failedOrSkipped [][]resource.TestResult) string { + var s string + if len(failedOrSkipped) > 0 { + s += fmt.Sprint("Failures/Skipped:\n\n") + for _, failedGroup := range failedOrSkipped { + first := failedGroup[0] + header := header(first) + if header != "" { + s += fmt.Sprint(header) + } + for _, testResult := range failedGroup { + s += fmt.Sprintln(humanizeResult(testResult)) + } + s += fmt.Sprint("\n") + } + } + return s +} diff --git a/outputs/rspecish.go b/outputs/rspecish.go index 2515d0e83..ff96b5040 100644 --- a/outputs/rspecish.go +++ b/outputs/rspecish.go @@ -5,52 +5,43 @@ import ( "time" "github.com/aelsabbahy/goss/resource" - "github.com/fatih/color" ) type Rspecish struct{} func (r Rspecish) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) { testCount := 0 - var failed [][]resource.TestResult + var failedOrSkipped [][]resource.TestResult + var skipped, failed int for resultGroup := range results { - failedGroup := []resource.TestResult{} + failedOrSkippedGroup := []resource.TestResult{} for _, testResult := range resultGroup { - if testResult.Successful { + switch testResult.Result { + case resource.SUCCESS: fmt.Printf(green(".")) - } else { + case resource.SKIP: + fmt.Printf(yellow("S")) + failedOrSkippedGroup = append(failedOrSkippedGroup, testResult) + skipped++ + case resource.FAIL: fmt.Printf(red("F")) - failedGroup = append(failedGroup, testResult) + failedOrSkippedGroup = append(failedOrSkippedGroup, testResult) + failed++ } testCount++ } - if len(failedGroup) > 0 { - failed = append(failed, failedGroup) + if len(failedOrSkippedGroup) > 0 { + failedOrSkipped = append(failedOrSkipped, failedOrSkippedGroup) } } fmt.Print("\n\n") - if len(failed) > 0 { - fmt.Println("Failures:\n") - for _, failedGroup := range failed { - first := failedGroup[0] - header := header(first) - if header != "" { - fmt.Print(header) - } - for _, testResult := range failedGroup { - fmt.Println(humanizeResult(testResult)) - } - fmt.Print("\n") - } - } + fmt.Print(failedOrSkippedSummary(failedOrSkipped)) - fmt.Printf("Total Duration: %.3fs\n", time.Since(startTime).Seconds()) - if len(failed) > 0 { - color.Red("Count: %d, Failed: %d\n", testCount, len(failed)) + fmt.Print(summary(startTime, testCount, failed, skipped)) + if failed > 0 { return 1 } - color.Green("Count: %d, Failed: %d\n", testCount, len(failed)) return 0 } diff --git a/outputs/tap.go b/outputs/tap.go index 54c3612d4..1ce09c5ed 100644 --- a/outputs/tap.go +++ b/outputs/tap.go @@ -19,11 +19,16 @@ func (r Tap) Output(results <-chan []resource.TestResult, startTime time.Time) ( for resultGroup := range results { for _, testResult := range resultGroup { - if testResult.Successful { + switch testResult.Result { + case resource.SUCCESS: summary[testCount] = "ok " + strconv.Itoa(testCount+1) + " - " + humanizeResult2(testResult) + "\n" - } else { + case resource.FAIL: summary[testCount] = "not ok " + strconv.Itoa(testCount+1) + " - " + humanizeResult2(testResult) + "\n" failed++ + case resource.SKIP: + summary[testCount] = "ok " + strconv.Itoa(testCount+1) + " - # SKIP " + humanizeResult2(testResult) + "\n" + default: + panic(fmt.Sprintf("Unexpected Result Code: %v\n", testResult.Result)) } testCount++ } diff --git a/resource/addr.go b/resource/addr.go index 417d0dc1e..c37bbe2fe 100644 --- a/resource/addr.go +++ b/resource/addr.go @@ -6,11 +6,11 @@ import ( ) type Addr struct { - Title string `json:"title,omitempty" yaml:"title,omitempty"` - Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` - Address string `json:"-" yaml:"-"` - Reachable bool `json:"reachable" yaml:"reachable"` - Timeout int `json:"timeout" yaml:"timeout"` + Title string `json:"title,omitempty" yaml:"title,omitempty"` + Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` + Address string `json:"-" yaml:"-"` + Reachable matcher `json:"reachable" yaml:"reachable"` + Timeout int `json:"timeout" yaml:"timeout"` } func (a *Addr) ID() string { return a.Address } @@ -21,15 +21,14 @@ func (r *Addr) GetTitle() string { return r.Title } func (r *Addr) GetMeta() meta { return r.Meta } func (a *Addr) Validate(sys *system.System) []TestResult { + skip := false if a.Timeout == 0 { a.Timeout = 500 } sysAddr := sys.NewAddr(a.Address, sys, util.Config{Timeout: a.Timeout}) var results []TestResult - - results = append(results, ValidateValue(a, "reachable", a.Reachable, sysAddr.Reachable)) - + results = append(results, ValidateValue(a, "reachable", a.Reachable, sysAddr.Reachable, skip)) return results } diff --git a/resource/command.go b/resource/command.go index d4abff410..1e9fb890a 100644 --- a/resource/command.go +++ b/resource/command.go @@ -27,23 +27,21 @@ func (c *Command) GetTitle() string { return c.Title } func (c *Command) GetMeta() meta { return c.Meta } func (c *Command) Validate(sys *system.System) []TestResult { + skip := false if c.Timeout == 0 { c.Timeout = 10000 } sysCommand := sys.NewCommand(c.Command, sys, util.Config{Timeout: c.Timeout}) var results []TestResult - cExitStatus := deprecateAtoI(c.ExitStatus, fmt.Sprintf("%s: command.exit-status", c.Command)) - results = append(results, ValidateValue(c, "exit-status", cExitStatus, sysCommand.ExitStatus)) - + results = append(results, ValidateValue(c, "exit-status", cExitStatus, sysCommand.ExitStatus, skip)) if len(c.Stdout) > 0 { - results = append(results, ValidateContains(c, "stdout", c.Stdout, sysCommand.Stdout)) + results = append(results, ValidateContains(c, "stdout", c.Stdout, sysCommand.Stdout, skip)) } if len(c.Stderr) > 0 { - results = append(results, ValidateContains(c, "stderr", c.Stderr, sysCommand.Stderr)) + results = append(results, ValidateContains(c, "stderr", c.Stderr, sysCommand.Stderr, skip)) } - return results } diff --git a/resource/dns.go b/resource/dns.go index 372d5422b..329d6cb9b 100644 --- a/resource/dns.go +++ b/resource/dns.go @@ -9,7 +9,7 @@ type DNS struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Host string `json:"-" yaml:"-"` - Resolveable bool `json:"resolveable" yaml:"resolveable"` + Resolveable matcher `json:"resolveable" yaml:"resolveable"` Addrs matcher `json:"addrs,omitempty" yaml:"addrs,omitempty"` Timeout int `json:"timeout" yaml:"timeout"` } @@ -21,19 +21,20 @@ func (d *DNS) GetTitle() string { return d.Title } func (d *DNS) GetMeta() meta { return d.Meta } func (d *DNS) Validate(sys *system.System) []TestResult { + skip := false if d.Timeout == 0 { d.Timeout = 500 } sysDNS := sys.NewDNS(d.Host, sys, util.Config{Timeout: d.Timeout}) var results []TestResult - - results = append(results, ValidateValue(d, "resolveable", d.Resolveable, sysDNS.Resolveable)) - + results = append(results, ValidateValue(d, "resolveable", d.Resolveable, sysDNS.Resolveable, skip)) + if shouldSkip(results) { + skip = true + } if d.Addrs != nil { - results = append(results, ValidateValue(d, "addrs", d.Addrs, sysDNS.Addrs)) + results = append(results, ValidateValue(d, "addrs", d.Addrs, sysDNS.Addrs, skip)) } - return results } diff --git a/resource/file.go b/resource/file.go index 0df26a2e3..273ff77ca 100644 --- a/resource/file.go +++ b/resource/file.go @@ -9,7 +9,7 @@ type File struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Path string `json:"-" yaml:"-"` - Exists bool `json:"exists" yaml:"exists"` + Exists matcher `json:"exists" yaml:"exists"` Mode matcher `json:"mode,omitempty" yaml:"mode,omitempty"` Size matcher `json:"size,omitempty" yaml:"size,omitempty"` Owner matcher `json:"owner,omitempty" yaml:"owner,omitempty"` @@ -26,40 +26,35 @@ func (f *File) GetTitle() string { return f.Title } func (f *File) GetMeta() meta { return f.Meta } func (f *File) Validate(sys *system.System) []TestResult { + skip := false sysFile := sys.NewFile(f.Path, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(f, "exists", f.Exists, sysFile.Exists)) - + results = append(results, ValidateValue(f, "exists", f.Exists, sysFile.Exists, skip)) + if shouldSkip(results) { + skip = true + } if f.Mode != nil { - results = append(results, ValidateValue(f, "mode", f.Mode, sysFile.Mode)) + results = append(results, ValidateValue(f, "mode", f.Mode, sysFile.Mode, skip)) } - if f.Owner != nil { - results = append(results, ValidateValue(f, "owner", f.Owner, sysFile.Owner)) + results = append(results, ValidateValue(f, "owner", f.Owner, sysFile.Owner, skip)) } - if f.Group != nil { - results = append(results, ValidateValue(f, "group", f.Group, sysFile.Group)) + results = append(results, ValidateValue(f, "group", f.Group, sysFile.Group, skip)) } - if f.LinkedTo != nil { - results = append(results, ValidateValue(f, "linkedto", f.LinkedTo, sysFile.LinkedTo)) + results = append(results, ValidateValue(f, "linkedto", f.LinkedTo, sysFile.LinkedTo, skip)) } - if f.Filetype != nil { - results = append(results, ValidateValue(f, "filetype", f.Filetype, sysFile.Filetype)) + results = append(results, ValidateValue(f, "filetype", f.Filetype, sysFile.Filetype, skip)) } - if len(f.Contains) > 0 { - results = append(results, ValidateContains(f, "contains", f.Contains, sysFile.Contains)) + results = append(results, ValidateContains(f, "contains", f.Contains, sysFile.Contains, skip)) } - if f.Size != nil { - results = append(results, ValidateValue(f, "size", f.Size, sysFile.Size)) + results = append(results, ValidateValue(f, "size", f.Size, sysFile.Size, skip)) } - return results } diff --git a/resource/gomega.go b/resource/gomega.go index a0e3e3fdf..85748ea16 100644 --- a/resource/gomega.go +++ b/resource/gomega.go @@ -23,6 +23,9 @@ func matcherToGomegaMatcher(matcher interface{}) (types.GomegaMatcher, error) { return gomega.And(matchers...), nil } matcher = sanitizeExpectedValue(matcher) + if matcher == nil { + return nil, fmt.Errorf("Missing Required Attribute") + } matcherMap, ok := matcher.(map[string]interface{}) if !ok { panic(fmt.Sprintf("Unexpected matcher type: %T\n\n", matcher)) diff --git a/resource/group.go b/resource/group.go index dd56dfb7a..a762701f3 100644 --- a/resource/group.go +++ b/resource/group.go @@ -11,7 +11,7 @@ type Group struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Groupname string `json:"-" yaml:"-"` - Exists bool `json:"exists" yaml:"exists"` + Exists matcher `json:"exists" yaml:"exists"` GID matcher `json:"gid,omitempty" yaml:"gid,omitempty"` } @@ -22,17 +22,18 @@ func (g *Group) GetTitle() string { return g.Title } func (g *Group) GetMeta() meta { return g.Meta } func (g *Group) Validate(sys *system.System) []TestResult { + skip := false sysgroup := sys.NewGroup(g.Groupname, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(g, "exists", g.Exists, sysgroup.Exists)) - + results = append(results, ValidateValue(g, "exists", g.Exists, sysgroup.Exists, skip)) + if shouldSkip(results) { + skip = true + } if g.GID != nil { gGID := deprecateAtoI(g.GID, fmt.Sprintf("%s: group.gid", g.Groupname)) - results = append(results, ValidateValue(g, "gid", gGID, sysgroup.GID)) + results = append(results, ValidateValue(g, "gid", gGID, sysgroup.GID, skip)) } - return results } diff --git a/resource/interface.go b/resource/interface.go index a851b6b2c..07711ad96 100644 --- a/resource/interface.go +++ b/resource/interface.go @@ -9,7 +9,7 @@ type Interface struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Name string `json:"-" yaml:"-"` - Exists bool `json:"exists" yaml:"exists"` + Exists matcher `json:"exists" yaml:"exists"` Addrs matcher `json:"addrs,omitempty" yaml:"addrs,omitempty"` } @@ -21,15 +21,17 @@ func (i *Interface) GetTitle() string { return i.Title } func (i *Interface) GetMeta() meta { return i.Meta } func (i *Interface) Validate(sys *system.System) []TestResult { + skip := false sysInterface := sys.NewInterface(i.Name, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(i, "exists", i.Exists, sysInterface.Exists)) + results = append(results, ValidateValue(i, "exists", i.Exists, sysInterface.Exists, skip)) + if shouldSkip(results) { + skip = true + } if i.Addrs != nil { - results = append(results, ValidateValue(i, "addrs", i.Addrs, sysInterface.Addrs)) + results = append(results, ValidateValue(i, "addrs", i.Addrs, sysInterface.Addrs, skip)) } - return results } diff --git a/resource/kernel_param.go b/resource/kernel_param.go index 48d9cde25..7e82c9caf 100644 --- a/resource/kernel_param.go +++ b/resource/kernel_param.go @@ -6,10 +6,10 @@ import ( ) type KernelParam struct { - Title string `json:"title,omitempty" yaml:"title,omitempty"` - Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` - Key string `json:"-" yaml:"-"` - Value string `json:"value" yaml:"value"` + Title string `json:"title,omitempty" yaml:"title,omitempty"` + Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` + Key string `json:"-" yaml:"-"` + Value matcher `json:"value" yaml:"value"` } func (a *KernelParam) ID() string { return a.Key } @@ -20,12 +20,11 @@ func (r *KernelParam) GetTitle() string { return r.Title } func (r *KernelParam) GetMeta() meta { return r.Meta } func (a *KernelParam) Validate(sys *system.System) []TestResult { + skip := false sysKernelParam := sys.NewKernelParam(a.Key, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(a, "value", a.Value, sysKernelParam.Value)) - + results = append(results, ValidateValue(a, "value", a.Value, sysKernelParam.Value, skip)) return results } diff --git a/resource/mount.go b/resource/mount.go index b588d4bc8..206eb2a27 100644 --- a/resource/mount.go +++ b/resource/mount.go @@ -9,7 +9,7 @@ type Mount struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` MountPoint string `json:"-" yaml:"-"` - Exists bool `json:"exists" yaml:"exists"` + Exists matcher `json:"exists" yaml:"exists"` Opts matcher `json:"opts,omitempty" yaml:"opts,omitempty"` Source matcher `json:"source,omitempty" yaml:"source,omitempty"` Filesystem matcher `json:"filesystem,omitempty" yaml:"filesystem,omitempty"` @@ -23,21 +23,23 @@ func (m *Mount) GetTitle() string { return m.Title } func (m *Mount) GetMeta() meta { return m.Meta } func (m *Mount) Validate(sys *system.System) []TestResult { + skip := false sysMount := sys.NewMount(m.MountPoint, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(m, "exists", m.Exists, sysMount.Exists)) + results = append(results, ValidateValue(m, "exists", m.Exists, sysMount.Exists, skip)) + if shouldSkip(results) { + skip = true + } if m.Opts != nil { - results = append(results, ValidateValue(m, "opts", m.Opts, sysMount.Opts)) + results = append(results, ValidateValue(m, "opts", m.Opts, sysMount.Opts, skip)) } if m.Source != nil { - results = append(results, ValidateValue(m, "source", m.Source, sysMount.Source)) + results = append(results, ValidateValue(m, "source", m.Source, sysMount.Source, skip)) } if m.Filesystem != nil { - results = append(results, ValidateValue(m, "filesystem", m.Filesystem, sysMount.Filesystem)) + results = append(results, ValidateValue(m, "filesystem", m.Filesystem, sysMount.Filesystem, skip)) } - return results } diff --git a/resource/package.go b/resource/package.go index 523055cff..b5170400f 100644 --- a/resource/package.go +++ b/resource/package.go @@ -9,7 +9,7 @@ type Package struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Name string `json:"-" yaml:"-"` - Installed bool `json:"installed" yaml:"installed"` + Installed matcher `json:"installed" yaml:"installed"` Versions matcher `json:"versions,omitempty" yaml:"versions,omitempty"` } @@ -20,16 +20,17 @@ func (p *Package) GetTitle() string { return p.Title } func (p *Package) GetMeta() meta { return p.Meta } func (p *Package) Validate(sys *system.System) []TestResult { + skip := false sysPkg := sys.NewPackage(p.Name, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(p, "installed", p.Installed, sysPkg.Installed)) - + results = append(results, ValidateValue(p, "installed", p.Installed, sysPkg.Installed, skip)) + if shouldSkip(results) { + skip = true + } if p.Versions != nil { - results = append(results, ValidateValue(p, "version", p.Versions, sysPkg.Versions)) + results = append(results, ValidateValue(p, "version", p.Versions, sysPkg.Versions, skip)) } - return results } diff --git a/resource/port.go b/resource/port.go index dd0b982f6..49873b313 100644 --- a/resource/port.go +++ b/resource/port.go @@ -9,7 +9,7 @@ type Port struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Port string `json:"-" yaml:"-"` - Listening bool `json:"listening" yaml:"listening"` + Listening matcher `json:"listening" yaml:"listening"` IP matcher `json:"ip,omitempty" yaml:"ip,omitempty"` } @@ -20,16 +20,17 @@ func (p *Port) GetTitle() string { return p.Title } func (p *Port) GetMeta() meta { return p.Meta } func (p *Port) Validate(sys *system.System) []TestResult { + skip := false sysPort := sys.NewPort(p.Port, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(p, "listening", p.Listening, sysPort.Listening)) - + results = append(results, ValidateValue(p, "listening", p.Listening, sysPort.Listening, skip)) + if shouldSkip(results) { + skip = true + } if p.IP != nil { - results = append(results, ValidateValue(p, "ip", p.IP, sysPort.IP)) + results = append(results, ValidateValue(p, "ip", p.IP, sysPort.IP, skip)) } - return results } diff --git a/resource/process.go b/resource/process.go index da226a8d8..9280eb128 100644 --- a/resource/process.go +++ b/resource/process.go @@ -6,10 +6,10 @@ import ( ) type Process struct { - Title string `json:"title,omitempty" yaml:"title,omitempty"` - Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` - Executable string `json:"-" yaml:"-"` - Running bool `json:"running" yaml:"running"` + Title string `json:"title,omitempty" yaml:"title,omitempty"` + Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` + Executable string `json:"-" yaml:"-"` + Running matcher `json:"running" yaml:"running"` } func (p *Process) ID() string { return p.Executable } @@ -19,12 +19,11 @@ func (p *Process) GetTitle() string { return p.Title } func (p *Process) GetMeta() meta { return p.Meta } func (p *Process) Validate(sys *system.System) []TestResult { + skip := false sysProcess := sys.NewProcess(p.Executable, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(p, "running", p.Running, sysProcess.Running)) - + results = append(results, ValidateValue(p, "running", p.Running, sysProcess.Running, skip)) return results } diff --git a/resource/resource.go b/resource/resource.go index c5ebe030b..2779776a0 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -57,3 +57,10 @@ func validAttrs(i interface{}, t string) (map[string]bool, error) { } return validAttrs, nil } + +func shouldSkip(results []TestResult) bool { + if results[0].Err != nil || results[0].Found[0] == "false" { + return true + } + return false +} diff --git a/resource/service.go b/resource/service.go index 655f741c9..2290b7a5b 100644 --- a/resource/service.go +++ b/resource/service.go @@ -6,11 +6,11 @@ import ( ) type Service struct { - Title string `json:"title,omitempty" yaml:"title,omitempty"` - Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` - Service string `json:"-" yaml:"-"` - Enabled bool `json:"enabled" yaml:"enabled"` - Running bool `json:"running" yaml:"running"` + Title string `json:"title,omitempty" yaml:"title,omitempty"` + Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` + Service string `json:"-" yaml:"-"` + Enabled matcher `json:"enabled" yaml:"enabled"` + Running matcher `json:"running" yaml:"running"` } func (s *Service) ID() string { return s.Service } @@ -20,13 +20,12 @@ func (s *Service) GetTitle() string { return s.Title } func (s *Service) GetMeta() meta { return s.Meta } func (s *Service) Validate(sys *system.System) []TestResult { + skip := false sysservice := sys.NewService(s.Service, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(s, "enabled", s.Enabled, sysservice.Enabled)) - results = append(results, ValidateValue(s, "running", s.Running, sysservice.Running)) - + results = append(results, ValidateValue(s, "enabled", s.Enabled, sysservice.Enabled, skip)) + results = append(results, ValidateValue(s, "running", s.Running, sysservice.Running, skip)) return results } diff --git a/resource/user.go b/resource/user.go index c8d29cdb1..54cb5d3f5 100644 --- a/resource/user.go +++ b/resource/user.go @@ -11,7 +11,7 @@ type User struct { Title string `json:"title,omitempty" yaml:"title,omitempty"` Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"` Username string `json:"-" yaml:"-"` - Exists bool `json:"exists" yaml:"exists"` + Exists matcher `json:"exists" yaml:"exists"` UID matcher `json:"uid,omitempty" yaml:"uid,omitempty"` GID matcher `json:"gid,omitempty" yaml:"gid,omitempty"` Groups matcher `json:"groups,omitempty" yaml:"groups,omitempty"` @@ -26,30 +26,31 @@ func (u *User) GetTitle() string { return u.Title } func (u *User) GetMeta() meta { return u.Meta } func (u *User) Validate(sys *system.System) []TestResult { + skip := false sysuser := sys.NewUser(u.Username, sys, util.Config{}) var results []TestResult - - results = append(results, ValidateValue(u, "exists", u.Exists, sysuser.Exists)) - + results = append(results, ValidateValue(u, "exists", u.Exists, sysuser.Exists, skip)) + if shouldSkip(results) { + skip = true + } if u.UID != nil { uUID := deprecateAtoI(u.UID, fmt.Sprintf("%s: user.uid", u.Username)) - results = append(results, ValidateValue(u, "uid", uUID, sysuser.UID)) + results = append(results, ValidateValue(u, "uid", uUID, sysuser.UID, skip)) } if u.GID != nil { uGID := deprecateAtoI(u.GID, fmt.Sprintf("%s: user.gid", u.Username)) - results = append(results, ValidateValue(u, "gid", uGID, sysuser.GID)) + results = append(results, ValidateValue(u, "gid", uGID, sysuser.GID, skip)) } if u.Home != nil { - results = append(results, ValidateValue(u, "home", u.Home, sysuser.Home)) + results = append(results, ValidateValue(u, "home", u.Home, sysuser.Home, skip)) } if u.Groups != nil { - results = append(results, ValidateValue(u, "groups", u.Groups, sysuser.Groups)) + results = append(results, ValidateValue(u, "groups", u.Groups, sysuser.Groups, skip)) } if u.Shell != nil { - results = append(results, ValidateValue(u, "shell", u.Shell, sysuser.Shell)) + results = append(results, ValidateValue(u, "shell", u.Shell, sysuser.Shell, skip)) } - return results } diff --git a/resource/validate.go b/resource/validate.go index f89471d9d..294a7e8d1 100644 --- a/resource/validate.go +++ b/resource/validate.go @@ -19,6 +19,12 @@ const ( Contains ) +const ( + SUCCESS = iota + FAIL + SKIP +) + type TestResult struct { Successful bool `json:"successful" yaml:"successful"` ResourceId string `json:"resource-id" yaml:"resource-id"` @@ -26,6 +32,7 @@ type TestResult struct { Title string `json:"title" yaml:"title"` Meta meta `json:"meta" yaml:"meta"` TestType int `json:"test-type" yaml:"test-type"` + Result int `json:"result" yaml:"result"` Property string `json:"property" yaml:"property"` Err error `json:"err" yaml:"err"` Expected []string `json:"expected" yaml:"expected"` @@ -34,13 +41,38 @@ type TestResult struct { Duration time.Duration `json:"duration" yaml:"duration"` } -func ValidateValue(res ResourceRead, property string, expectedValue interface{}, actual interface{}) TestResult { +func skipResult(typeS string, testType int, id string, title string, meta meta, property string, startTime time.Time) TestResult { + return TestResult{ + Successful: true, + Result: SKIP, + ResourceType: typeS, + TestType: testType, + ResourceId: id, + Title: title, + Meta: meta, + Property: property, + Duration: startTime.Sub(startTime), + } +} + +func ValidateValue(res ResourceRead, property string, expectedValue interface{}, actual interface{}, skip bool) TestResult { id := res.ID() title := res.GetTitle() meta := res.GetMeta() typ := reflect.TypeOf(res) - typs := strings.Split(typ.String(), ".")[1] + typeS := strings.Split(typ.String(), ".")[1] startTime := time.Now() + if skip { + return skipResult( + typeS, + Values, + id, + title, + meta, + property, + startTime, + ) + } var foundValue interface{} var err error @@ -71,7 +103,8 @@ func ValidateValue(res ResourceRead, property string, expectedValue interface{}, if err != nil { return TestResult{ Successful: false, - ResourceType: typs, + Result: FAIL, + ResourceType: typeS, TestType: Values, ResourceId: id, Title: title, @@ -83,8 +116,10 @@ func ValidateValue(res ResourceRead, property string, expectedValue interface{}, } var failMessage string + var result int if !success { failMessage = gomegaMatcher.FailureMessage(foundValue) + result = FAIL } expected, _ := json.Marshal(expectedValue) @@ -92,7 +127,8 @@ func ValidateValue(res ResourceRead, property string, expectedValue interface{}, return TestResult{ Successful: success, - ResourceType: typs, + Result: result, + ResourceType: typeS, TestType: Value, ResourceId: id, Title: title, @@ -207,13 +243,24 @@ func patternsToSlice(patterns []patternMatcher) []string { return slice } -func ValidateContains(res ResourceRead, property string, expectedValues []string, method func() (io.Reader, error)) TestResult { +func ValidateContains(res ResourceRead, property string, expectedValues []string, method func() (io.Reader, error), skip bool) TestResult { id := res.ID() title := res.GetTitle() meta := res.GetMeta() typ := reflect.TypeOf(res) - typs := strings.Split(typ.String(), ".")[1] + typeS := strings.Split(typ.String(), ".")[1] startTime := time.Now() + if skip { + return skipResult( + typeS, + Values, + id, + title, + meta, + property, + startTime, + ) + } var err error var fh io.Reader var notfound []patternMatcher @@ -224,7 +271,8 @@ func ValidateContains(res ResourceRead, property string, expectedValues []string if err != nil { return TestResult{ Successful: false, - ResourceType: typs, + Result: FAIL, + ResourceType: typeS, TestType: Contains, ResourceId: id, Title: title, @@ -259,7 +307,8 @@ func ValidateContains(res ResourceRead, property string, expectedValues []string if err := scanner.Err(); err != nil { return TestResult{ Successful: false, - ResourceType: typs, + Result: FAIL, + ResourceType: typeS, TestType: Contains, ResourceId: id, Title: title, @@ -281,7 +330,8 @@ func ValidateContains(res ResourceRead, property string, expectedValues []string if len(expectedValues) != len(found) { return TestResult{ Successful: false, - ResourceType: typs, + Result: FAIL, + ResourceType: typeS, TestType: Contains, ResourceId: id, Title: title, @@ -294,7 +344,8 @@ func ValidateContains(res ResourceRead, property string, expectedValues []string } return TestResult{ Successful: true, - ResourceType: typs, + Result: SUCCESS, + ResourceType: typeS, TestType: Contains, ResourceId: id, Title: title, diff --git a/resource/validate_test.go b/resource/validate_test.go index 3e92164c9..efa348f1f 100644 --- a/resource/validate_test.go +++ b/resource/validate_test.go @@ -16,7 +16,6 @@ func (f *FakeResource) ID() string { } func (f *FakeResource) GetTitle() string { return "title" } -//func (f *FakeResource) GetMeta() meta { return map[string]interface{}{"foo": "bar"} } func (f *FakeResource) GetMeta() meta { return meta{"foo": "bar"} } var stringTests = []struct { @@ -35,7 +34,7 @@ func TestValidateValue(t *testing.T) { inFunc := func() (interface{}, error) { return c.in2, nil } - got := ValidateValue(&FakeResource{""}, "", c.in, inFunc) + got := ValidateValue(&FakeResource{""}, "", c.in, inFunc, false) if got.Successful != c.want { t.Errorf("%+v: got %v, want %v", c, got.Successful, c.want) } @@ -47,19 +46,31 @@ func TestValidateValueErr(t *testing.T) { inFunc := func() (interface{}, error) { return c.in2, fmt.Errorf("some err") } - got := ValidateValue(&FakeResource{""}, "", c.in, inFunc) + got := ValidateValue(&FakeResource{""}, "", c.in, inFunc, false) if got.Successful != false { t.Errorf("%+v: got %v, want %v", c, got.Successful, false) } } } +func TestValidateValueSkip(t *testing.T) { + for _, c := range stringTests { + inFunc := func() (interface{}, error) { + return c.in2, nil + } + got := ValidateValue(&FakeResource{""}, "", c.in, inFunc, true) + if got.Result != SKIP { + t.Errorf("%+v: got %v, want %v", c, got.Result, SKIP) + } + } +} + func BenchmarkValidateValue(b *testing.B) { inFunc := func() (interface{}, error) { return "foo", nil } for n := 0; n < b.N; n++ { - ValidateValue(&FakeResource{""}, "", "foo", inFunc) + ValidateValue(&FakeResource{""}, "", "foo", inFunc, false) } } @@ -85,7 +96,7 @@ func TestValidateContains(t *testing.T) { reader := strings.NewReader(c.in2) return reader, nil } - got := ValidateContains(&FakeResource{""}, "", c.in, inFunc) + got := ValidateContains(&FakeResource{""}, "", c.in, inFunc, false) if got.Successful != c.want { t.Errorf("%+v: got %v, want %v", c, got.Successful, c.want) } @@ -98,7 +109,7 @@ func TestValidateContainsErr(t *testing.T) { reader := strings.NewReader(c.in2) return reader, fmt.Errorf("some err") } - got := ValidateContains(&FakeResource{""}, "", c.in, inFunc) + got := ValidateContains(&FakeResource{""}, "", c.in, inFunc, false) if got.Successful != false { t.Errorf("%+v: got %v, want %v", c, got.Successful, false) } @@ -110,8 +121,21 @@ func TestValidateContainsBadRegexErr(t *testing.T) { reader := strings.NewReader("dummy") return reader, nil } - got := ValidateContains(&FakeResource{""}, "", []string{"/*\\.* @@.*/"}, inFunc) + got := ValidateContains(&FakeResource{""}, "", []string{"/*\\.* @@.*/"}, inFunc, false) if got.Err == nil { t.Errorf("Expected bad regex to raise error, got nil") } } + +func TestValidateContainsSkip(t *testing.T) { + for _, c := range containsTests { + inFunc := func() (io.Reader, error) { + reader := strings.NewReader(c.in2) + return reader, nil + } + got := ValidateContains(&FakeResource{""}, "", c.in, inFunc, true) + if got.Result != SKIP { + t.Errorf("%+v: got %v, want %v", c, got.Result, SKIP) + } + } +}