Skip to content

Commit

Permalink
Add concept of skipped tests
Browse files Browse the repository at this point in the history 
* Skip optional tests for non existing system resources
* Skipped tests do not count as a failure on their own, but are almost
always accompanied with a failing test, the only exception is when the
exists test is made optional through matchers (see #107):
```yaml
exists:
  or:
  - true
  - false
```
* Undefined required attributes will now result in an error
  • Loading branch information
@aelsabbahy
aelsabbahy committed Jul 24, 2016
1 parent 5fa57d7 commit 0c5e0ff
Show file tree
Hide file tree
Showing 23 changed files with 322 additions and 210 deletions.
46 changes: 18 additions & 28 deletions outputs/documentation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,59 +5,49 @@ import (
"time"

"github.com/aelsabbahy/goss/resource"
"github.com/fatih/color"
)

type Documentation struct{}

func (r Documentation) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) {
testCount := 0
var failed [][]resource.TestResult
var failedOrSkipped [][]resource.TestResult
var skipped, failed int
for resultGroup := range results {
failedGroup := []resource.TestResult{}
failedOrSkippedGroup := []resource.TestResult{}
first := resultGroup[0]
header := header(first)
if header != "" {
fmt.Print(header)
}
for _, testResult := range resultGroup {
if testResult.Successful {
switch testResult.Result {
case resource.SUCCESS:
fmt.Println(humanizeResult(testResult))
testCount++
} else {
case resource.SKIP:
fmt.Println(humanizeResult(testResult))
failedGroup = append(failedGroup, testResult)
testCount++
failedOrSkippedGroup = append(failedOrSkippedGroup, testResult)
skipped++
case resource.FAIL:
fmt.Println(humanizeResult(testResult))
failedOrSkippedGroup = append(failedOrSkippedGroup, testResult)
failed++
}
testCount++
}
fmt.Println("")
if len(failedGroup) > 0 {
failed = append(failed, failedGroup)
if len(failedOrSkippedGroup) > 0 {
failedOrSkipped = append(failedOrSkipped, failedOrSkippedGroup)
}
}

fmt.Print("\n\n")
if len(failed) > 0 {
fmt.Println("Failures:\n")
for _, failedGroup := range failed {
first := failedGroup[0]
header := header(first)
if header != "" {
fmt.Print(header)
}
for _, testResult := range failedGroup {
fmt.Println(humanizeResult(testResult))
}
fmt.Print("\n")
}
}
fmt.Print(failedOrSkippedSummary(failedOrSkipped))

fmt.Printf("Total Duration: %.3fs\n", time.Since(startTime).Seconds())
if len(failed) > 0 {
color.Red("Count: %d, Failed: %d\n", testCount, len(failed))
fmt.Print(summary(startTime, testCount, failed, skipped))
if failed > 0 {
return 1
}
color.Green("Count: %d, Failed: %d\n", testCount, len(failed))
return 0
}

Expand Down
28 changes: 14 additions & 14 deletions outputs/junit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@ import (
type JUnit struct{}

func (r JUnit) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) {
testCount := 0
failed := 0
var testCount, failed, skipped int

// ISO8601 timeformat
location, _ := time.LoadLocation("Etc/UTC")
Expand All @@ -30,22 +29,23 @@ func (r JUnit) Output(results <-chan []resource.TestResult, startTime time.Time)
testResult.ResourceId + " " +
testResult.Property + "\" " +
"time=\"" + duration + "\">\n"
if testResult.Successful {
summary[testCount] = summary[testCount] +
"<system-out>" +
humanizeResult2(testResult) +
"</system-out>\n</testcase>\n"
} else {
summary[testCount] = summary[testCount] +
"<system-err>" +
if testResult.Result == resource.FAIL {
summary[testCount] += "<system-err>" +
humanizeResult2(testResult) +
"</system-err>\n"
summary[testCount] = summary[testCount] +
"<failure>" +
summary[testCount] += "<failure>" +
humanizeResult2(testResult) +
"</failure>\n</testcase>\n"

failed++
} else {
if testResult.Result == resource.SKIP {
summary[testCount] += "<skipped/>"
skipped++
}
summary[testCount] += "<system-out>" +
humanizeResult2(testResult) +
"</system-out>\n</testcase>\n"
}
testCount++
}
Expand All @@ -54,8 +54,8 @@ func (r JUnit) Output(results <-chan []resource.TestResult, startTime time.Time)
duration := time.Since(startTime)
fmt.Println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
fmt.Printf("<testsuite name=\"goss\" errors=\"0\" tests=\"%d\" "+
"failures=\"%d\" time=\"%.3f\" timestamp=\"%s\">\n",
testCount, failed, duration.Seconds(), timestamp)
"failures=\"%d\" skipped=\"%d\" time=\"%.3f\" timestamp=\"%s\">\n",
testCount, failed, skipped, duration.Seconds(), timestamp)

for i := 0; i < testCount; i++ {
fmt.Printf("%s", summary[i])
Expand Down
12 changes: 7 additions & 5 deletions outputs/nagios.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,25 @@ import (
type Nagios struct{}

func (r Nagios) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) {
testCount := 0
failed := 0
var testCount, failed, skipped int
for resultGroup := range results {
for _, testResult := range resultGroup {
if !testResult.Successful {
switch testResult.Result {
case resource.FAIL:
failed++
case resource.SKIP:
skipped++
}
testCount++
}
}

duration := time.Since(startTime)
if failed > 0 {
fmt.Printf("GOSS CRITICAL - Count: %d, Failed: %d, Duration: %.3fs\n", testCount, failed, duration.Seconds())
fmt.Printf("GOSS CRITICAL - Count: %d, Failed: %d, Skipped: %d, Duration: %.3fs\n", testCount, failed, skipped, duration.Seconds())
return 2
}
fmt.Printf("GOSS OK - Count: %d, Failed: %d, Duration: %.3fs\n", testCount, failed, duration.Seconds())
fmt.Printf("GOSS OK - Count: %d, Failed: %d, Skipped: %d, Duration: %.3fs\n", testCount, failed, skipped, duration.Seconds())
return 0
}

Expand Down
75 changes: 58 additions & 17 deletions outputs/outputs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,25 @@ type Outputer interface {

var green = color.New(color.FgGreen).SprintfFunc()
var red = color.New(color.FgRed).SprintfFunc()
var yellow = color.New(color.FgYellow).SprintfFunc()

func humanizeResult(r resource.TestResult) string {
if r.Err != nil {
return red("%s: %s: Error: %s", r.ResourceId, r.Property, r.Err)
}

if r.Successful {
switch r.Result {
case resource.SUCCESS:
return green("%s: %s: %s: matches expectation: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected)
} else {
case resource.SKIP:
return yellow("%s: %s: %s: skipped", r.ResourceType, r.ResourceId, r.Property)
case resource.FAIL:
if r.Human != "" {
return red("%s: %s: %s:\n%s", r.ResourceType, r.ResourceId, r.Property, r.Human)
}
return humanizeResult2(r)
default:
panic(fmt.Sprintf("Unexpected Result Code: %v\n", r.Result))
}
}

Expand All @@ -39,27 +45,33 @@ func humanizeResult2(r resource.TestResult) string {
return red("%s: %s: Error: %s", r.ResourceId, r.Property, r.Err)
}

switch r.TestType {
case resource.Value:
if r.Successful {
switch r.Result {
case resource.SUCCESS:
switch r.TestType {
case resource.Value:
return green("%s: %s: %s: matches expectation: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected)
} else {
return red("%s: %s: %s: doesn't match, expect: %s found: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected, r.Found)
}
case resource.Values:
if r.Successful {
case resource.Values:
return green("%s: %s: %s: all expectations found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(r.Expected, ", "))
} else {
return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", "))
case resource.Contains:
return green("%s: %s: %s: all expectations found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(r.Expected, ", "))
default:
return red("Unexpected type %d", r.TestType)
}
case resource.Contains:
if r.Successful {
return green("%s: %s: %s: all patterns found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(r.Expected, ", "))
} else {
case resource.FAIL:
switch r.TestType {
case resource.Value:
return red("%s: %s: %s: doesn't match, expect: %s found: %s", r.ResourceType, r.ResourceId, r.Property, r.Expected, r.Found)
case resource.Values:
return red("%s: %s: %s: expectations not found [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", "))
case resource.Contains:
return red("%s: %s: %s: patterns not found: [%s]", r.ResourceType, r.ResourceId, r.Property, strings.Join(subtractSlice(r.Expected, r.Found), ", "))
default:
return red("Unexpected type %d", r.TestType)
}
case resource.SKIP:
return yellow("%s: %s: %s: skipped", r.ResourceType, r.ResourceId, r.Property)
default:
return red("Unexpected type %d", r.TestType)
panic(fmt.Sprintf("Unexpected Result Code: %v\n", r.Result))
}
}

Expand Down Expand Up @@ -139,3 +151,32 @@ func header(t resource.TestResult) string {
}
return out
}

func summary(startTime time.Time, count, failed, skipped int) string {
var s string
s += fmt.Sprintf("Total Duration: %.3fs\n", time.Since(startTime).Seconds())
f := green
if failed > 0 {
f = red
}
s += f("Count: %d, Failed: %d, Skipped: %d\n", count, failed, skipped)
return s
}
func failedOrSkippedSummary(failedOrSkipped [][]resource.TestResult) string {
var s string
if len(failedOrSkipped) > 0 {
s += fmt.Sprint("Failures/Skipped:\n\n")
for _, failedGroup := range failedOrSkipped {
first := failedGroup[0]
header := header(first)
if header != "" {
s += fmt.Sprint(header)
}
for _, testResult := range failedGroup {
s += fmt.Sprintln(humanizeResult(testResult))
}
s += fmt.Sprint("\n")
}
}
return s
}
43 changes: 17 additions & 26 deletions outputs/rspecish.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,52 +5,43 @@ import (
"time"

"github.com/aelsabbahy/goss/resource"
"github.com/fatih/color"
)

type Rspecish struct{}

func (r Rspecish) Output(results <-chan []resource.TestResult, startTime time.Time) (exitCode int) {
testCount := 0
var failed [][]resource.TestResult
var failedOrSkipped [][]resource.TestResult
var skipped, failed int
for resultGroup := range results {
failedGroup := []resource.TestResult{}
failedOrSkippedGroup := []resource.TestResult{}
for _, testResult := range resultGroup {
if testResult.Successful {
switch testResult.Result {
case resource.SUCCESS:
fmt.Printf(green("."))
} else {
case resource.SKIP:
fmt.Printf(yellow("S"))
failedOrSkippedGroup = append(failedOrSkippedGroup, testResult)
skipped++
case resource.FAIL:
fmt.Printf(red("F"))
failedGroup = append(failedGroup, testResult)
failedOrSkippedGroup = append(failedOrSkippedGroup, testResult)
failed++
}
testCount++
}
if len(failedGroup) > 0 {
failed = append(failed, failedGroup)
if len(failedOrSkippedGroup) > 0 {
failedOrSkipped = append(failedOrSkipped, failedOrSkippedGroup)
}
}

fmt.Print("\n\n")
if len(failed) > 0 {
fmt.Println("Failures:\n")
for _, failedGroup := range failed {
first := failedGroup[0]
header := header(first)
if header != "" {
fmt.Print(header)
}
for _, testResult := range failedGroup {
fmt.Println(humanizeResult(testResult))
}
fmt.Print("\n")
}
}
fmt.Print(failedOrSkippedSummary(failedOrSkipped))

fmt.Printf("Total Duration: %.3fs\n", time.Since(startTime).Seconds())
if len(failed) > 0 {
color.Red("Count: %d, Failed: %d\n", testCount, len(failed))
fmt.Print(summary(startTime, testCount, failed, skipped))
if failed > 0 {
return 1
}
color.Green("Count: %d, Failed: %d\n", testCount, len(failed))
return 0
}

Expand Down
9 changes: 7 additions & 2 deletions outputs/tap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,16 @@ func (r Tap) Output(results <-chan []resource.TestResult, startTime time.Time) (

for resultGroup := range results {
for _, testResult := range resultGroup {
if testResult.Successful {
switch testResult.Result {
case resource.SUCCESS:
summary[testCount] = "ok " + strconv.Itoa(testCount+1) + " - " + humanizeResult2(testResult) + "\n"
} else {
case resource.FAIL:
summary[testCount] = "not ok " + strconv.Itoa(testCount+1) + " - " + humanizeResult2(testResult) + "\n"
failed++
case resource.SKIP:
summary[testCount] = "ok " + strconv.Itoa(testCount+1) + " - # SKIP " + humanizeResult2(testResult) + "\n"
default:
panic(fmt.Sprintf("Unexpected Result Code: %v\n", testResult.Result))
}
testCount++
}
Expand Down
15 changes: 7 additions & 8 deletions resource/addr.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ import (
)

type Addr struct {
Title string `json:"title,omitempty" yaml:"title,omitempty"`
Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"`
Address string `json:"-" yaml:"-"`
Reachable bool `json:"reachable" yaml:"reachable"`
Timeout int `json:"timeout" yaml:"timeout"`
Title string `json:"title,omitempty" yaml:"title,omitempty"`
Meta meta `json:"meta,omitempty" yaml:"meta,omitempty"`
Address string `json:"-" yaml:"-"`
Reachable matcher `json:"reachable" yaml:"reachable"`
Timeout int `json:"timeout" yaml:"timeout"`
}

func (a *Addr) ID() string { return a.Address }
Expand All @@ -21,15 +21,14 @@ func (r *Addr) GetTitle() string { return r.Title }
func (r *Addr) GetMeta() meta { return r.Meta }

func (a *Addr) Validate(sys *system.System) []TestResult {
skip := false
if a.Timeout == 0 {
a.Timeout = 500
}
sysAddr := sys.NewAddr(a.Address, sys, util.Config{Timeout: a.Timeout})

var results []TestResult

results = append(results, ValidateValue(a, "reachable", a.Reachable, sysAddr.Reachable))

results = append(results, ValidateValue(a, "reachable", a.Reachable, sysAddr.Reachable, skip))
return results
}

Expand Down
Loading

0 comments on commit 0c5e0ff

Please sign in to comment.