Skip to content

Commit

Permalink
Enable user to hide skipped workflows in report
Browse files Browse the repository at this point in the history 
Skipped workflows can be hidden from the report by setting
`OCI_HIDE_SKIPPED_WORKFLOWS=1` in the environment.

Also improves whitespace organization in the HTML template string.

Signed-off-by: Peter Engelbert <[email protected]>
  • Loading branch information
@pmengelbert
pmengelbert committed Apr 9, 2020
1 parent 846feae commit dca82e7
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 125 deletions.
2 changes: 1 addition & 1 deletion conformance/01_pull_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
)

var test01Pull = func() {
g.Context("Pull", func() {
g.Context(titlePull, func() {

var tag string

Expand Down
2 changes: 1 addition & 1 deletion conformance/02_push_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

var test02Push = func() {
g.Context("Push", func() {
g.Context(titlePush, func() {

var lastResponse *reggie.Response

Expand Down
2 changes: 1 addition & 1 deletion conformance/03_discovery_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import (
)

var test03ContentDiscovery = func() {
g.Context("Content Discovery", func() {
g.Context(titleContentDiscovery, func() {

var numTags = 4
var tagList []string
Expand Down
2 changes: 1 addition & 1 deletion conformance/04_management_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

var test04ContentManagement = func() {
g.Context("Content Management", func() {
g.Context(titleContentManagement, func() {

const defaultTagName = "tagtest0"
var tagToDelete string
Expand Down
9 changes: 9 additions & 0 deletions conformance/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,15 @@ OCI_TEST_CONTENT_MANAGEMENT=1
Note: The Content Management tests explicitly depend upon the Push and Content Discovery tests, as there is no
way to test content management without also supporting push and content discovery.

#### HTML Report
By default, the HTML report will show tests from all workflows. To hide workflows that have been disabled from
the report, you must set the following in the environment:

```
# Required to hide disabled workflows
OCI_HIDE_SKIPPED_WORKFLOWS=1
```

#### Container Image

You may use the [Dockerfile](./Dockerfile) located in this directory
Expand Down
246 changes: 134 additions & 112 deletions conformance/reporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,115 +176,118 @@ const (
}
</script>
</head>
<body>
<h1>OCI Distribution Conformance Tests</h1>
<div class="summary">
<div class="summary-bullet">
<div class="bullet-left">Summary:</div>
<div class="quick-summary">
{{- if gt .SuiteSummary.NumberOfPassedSpecs 0 -}}
<span class="darkgreen">
{{- if .AllPassed -}}All {{ end -}}{{ .SuiteSummary.NumberOfPassedSpecs }} passed</span>
{{- end -}}
{{- if gt .SuiteSummary.NumberOfFailedSpecs 0 -}}
<span class="darkred">
{{- if .AllFailed -}}All {{ end -}}{{ .SuiteSummary.NumberOfFailedSpecs }} failed</span>
{{- end -}}
{{- if gt .SuiteSummary.NumberOfSkippedSpecs 0 -}}
<span class="darkgrey">
{{- if .AllSkipped -}}All {{ end -}}{{ .SuiteSummary.NumberOfSkippedSpecs }} skipped</span>
{{- end -}}
<div class="meter">
<div class="meter-green"></div><div class="meter-red"></div><div class="meter-grey"></div>
</div>
<body>
<h1>OCI Distribution Conformance Tests</h1>
<div class="summary">
<div class="summary-bullet">
<div class="bullet-left">Summary:</div>
<div class="quick-summary">
{{- if gt .SuiteSummary.NumberOfPassedSpecs 0 -}}
<span class="darkgreen">
{{- if .AllPassed -}}All {{ end -}}{{ .SuiteSummary.NumberOfPassedSpecs }} passed</span>
{{- end -}}
{{- if gt .SuiteSummary.NumberOfFailedSpecs 0 -}}
<span class="darkred">
{{- if .AllFailed -}}All {{ end -}}{{ .SuiteSummary.NumberOfFailedSpecs }} failed</span>
{{- end -}}
{{- if gt .SuiteSummary.NumberOfSkippedSpecs 0 -}}
<span class="darkgrey">
{{- if .AllSkipped -}}All {{ end -}}{{ .SuiteSummary.NumberOfSkippedSpecs }} skipped</span>
{{- end -}}
<div class="meter">
<div class="meter-green"></div>
<div class="meter-red"></div>
<div class="meter-grey"></div>
</div>
</div>
</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Start time:</div>
<div class="bullet-right">{{ .StartTimeString }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">End time:</div>
<div class="bullet-right">{{ .EndTimeString }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Elapsed time:</div>
<div class="bullet-right">{{ .RunTime }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Test version:</div>
<div class="bullet-right">{{ .Version }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Configuration:</div>
<div class="bullet-right">
{{ range $i, $s := .EnvironmentVariables }}
{{ $s }}<br />
{{ end }}
<div class="summary-bullet">
<div class="bullet-left">Start time:</div>
<div class="bullet-right">{{ .StartTimeString }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">End time:</div>
<div class="bullet-right">{{ .EndTimeString }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Elapsed time:</div>
<div class="bullet-right">{{ .RunTime }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Test version:</div>
<div class="bullet-right">{{ .Version }}</div>
</div>
<div class="summary-bullet">
<div class="bullet-left">Configuration:</div>
<div class="bullet-right">
{{ range $i, $s := .EnvironmentVariables }}
{{ $s }}<br />
{{ end }}
</div>
</div>
</div>
</div>
<div>
{{with .Suite}}
{{$suite := .M}}
{{range $i, $suiteKey := .Keys}}
<h2>{{$suiteKey}}</h2>
<div class="subcategory">
{{$wf := index $suite $suiteKey}}
{{with $wf}}
{{$workflow := .M}}
{{range $j, $workflowKey := .Keys}}
<h3>{{$workflowKey}}</h3>
{{$ctg := index $workflow $workflowKey}}
{{with $ctg}}
{{$category := .M}}
{{range $k, $categoryKey := .Keys}}
{{$s := index $category $categoryKey}}
{{if eq $s.State 4}}
<div class="result red">
<div id="output-box-{{$s.ID}}-button" class="toggle"
onclick="javascript:toggleOutput('output-box-{{$s.ID}}')">+</div>
<h4 style="display: inline;">{{$s.Title}}</h4>
<br>
<div>
<div id="output-box-{{$s.ID}}" style="display: none;">
<pre class="pre-box">{{$s.CapturedOutput}}</pre>
</div>
</div>
<pre class="fail-message">{{$s.Failure.Message}}</pre>
<br>
</div>
{{else if eq $s.State 3}}
<div class="result green">
<div id="output-box-{{$s.ID}}-button" class="toggle"
onclick="javascript:toggleOutput('output-box-{{$s.ID}}')">+</div>
<h4 style="display: inline;">{{$s.Title}}</h4>
<br>
<div id="output-box-{{$s.ID}}" style="display: none;">
<pre class="pre-box">{{$s.CapturedOutput}}</pre>
</div>
</div>
{{else if eq $s.State 2}}
<div class="result grey">
<div id="output-box-{{$s.ID}}-button" class="toggle"
onclick="javascript:toggleOutput('output-box-{{$s.ID}}')">+</div>
<h4 style="display: inline;">{{$s.Title}}</h4>
<br>
<div id="output-box-{{$s.ID}}" style="display: none;">
<pre class="pre-box">{{$s.Failure.Message}}</pre>
</div>
</div>
{{end}}
{{end}}<br>
{{end}}
{{end}}
{{end}}
</div>
{{end}}
{{end}}
<div>
{{with .Suite}}
{{$suite := .M}}
{{range $i, $suiteKey := .Keys}}
{{$wf := index $suite $suiteKey}}
{{with $wf}}
{{ if .IsEnabled }}
<h2>{{$suiteKey}}</h2>
<div class="subcategory">
{{$workflow := .M}}
{{range $j, $workflowKey := .Keys}}
<h3>{{$workflowKey}}</h3>
{{$ctg := index $workflow $workflowKey}}
{{with $ctg}}
{{$category := .M}}
{{range $k, $categoryKey := .Keys}}
{{$s := index $category $categoryKey}}
{{if eq $s.State 4}}
<div class="result red">
<div id="output-box-{{$s.ID}}-button" class="toggle"
onclick="javascript:toggleOutput('output-box-{{$s.ID}}')">+</div>
<h4 style="display: inline;">{{$s.Title}}</h4>
<br>
<div>
<div id="output-box-{{$s.ID}}" style="display: none;">
<pre class="pre-box">{{$s.CapturedOutput}}</pre>
</div>
</div>
<pre class="fail-message">{{$s.Failure.Message}}</pre>
<br>
</div>
{{else if eq $s.State 3}}
<div class="result green">
<div id="output-box-{{$s.ID}}-button" class="toggle"
onclick="javascript:toggleOutput('output-box-{{$s.ID}}')">+</div>
<h4 style="display: inline;">{{$s.Title}}</h4>
<br>
<div id="output-box-{{$s.ID}}" style="display: none;">
<pre class="pre-box">{{$s.CapturedOutput}}</pre>
</div>
</div>
{{else if eq $s.State 2}}
<div class="result grey">
<div id="output-box-{{$s.ID}}-button" class="toggle"
onclick="javascript:toggleOutput('output-box-{{$s.ID}}')">+</div>
<h4 style="display: inline;">{{$s.Title}}</h4>
<br>
<div id="output-box-{{$s.ID}}" style="display: none;">
<pre class="pre-box">{{$s.Failure.Message}}</pre>
</div>
</div>
{{end}}
{{end}}<br>
{{end}}
{{end}}
{{end}}
{{end}}
</div>
{{end}}
{{end}}
</div>
</body>
</html>
</body></html>
`
)

Expand All @@ -302,8 +305,9 @@ type (
}

workflow struct {
M map[string]*category
Keys []string
M map[string]*category
IsEnabled bool
Keys []string
}

category struct {
Expand Down Expand Up @@ -340,6 +344,7 @@ type (
SuiteSummary *types.SuiteSummary
debugLogger *httpDebugWriter
debugIndex int
enabledMap map[string]bool
PercentPassed int
PercentFailed int
PercentSkipped int
Expand Down Expand Up @@ -431,10 +436,27 @@ func (l *httpDebugLogger) output(format string, v ...interface{}) {
}
}

func newHTMLReporter(htmlReportFilename string) *HTMLReporter {
func newHTMLReporter(htmlReportFilename string) (h *HTMLReporter) {
enabledMap := map[string]bool{
titlePull: true,
titlePush: true,
titleContentDiscovery: true,
titleContentManagement: true,
}

if os.Getenv(envVarHideSkippedWorkflows) == "1" {
enabledMap = map[string]bool{
titlePull: true,
titlePush: !userDisabled(push),
titleContentDiscovery: !userDisabled(contentDiscovery),
titleContentManagement: !userDisabled(contentManagement),
}
}

return &HTMLReporter{
htmlReportFilename: htmlReportFilename,
debugLogger: httpWriter,
enabledMap: enabledMap,
SpecSummaryMap: summaryMap{M: make(map[string]snapShotList)},
Suite: suite{
M: make(map[string]*workflow),
Expand All @@ -450,19 +472,19 @@ func (reporter *HTMLReporter) SpecDidComplete(specSummary *types.SpecSummary) {
}
specSummary.CapturedOutput = b.String()

//header := specSummary.ComponentTexts[categoryIndex]
snapshot := newSpecSnapshot(specSummary, reporter.Suite.Size)
reporter.Save(snapshot)
reporter.save(snapshot)
reporter.debugIndex = len(reporter.debugLogger.CapturedOutput)
}

func (reporter *HTMLReporter) Save(snapshot *specSnapshot) {
func (reporter *HTMLReporter) save(snapshot *specSnapshot) {
suite := &reporter.Suite
ct := snapshot.ComponentTexts
suiteName, categoryName, specTitle := ct[flowIndex], ct[categoryIndex], ct[specIndex]
//make the map of categories
if _, ok := suite.M[suiteName]; !ok {
suite.M[suiteName] = &workflow{M: make(map[string]*category), Keys: []string{}}
suite.M[suiteName] = &workflow{M: make(map[string]*category), Keys: []string{},
IsEnabled: reporter.enabledMap[suiteName]}
suite.Keys = append(suite.Keys, suiteName)
}
//make the map of snapshots
Expand Down
24 changes: 15 additions & 9 deletions conformance/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,21 @@ const (
DENIED
UNSUPPORTED

envTrue = "1"
envVarPush = "OCI_TEST_PUSH"
envVarContentDiscovery = "OCI_TEST_CONTENT_DISCOVERY"
envVarContentManagement = "OCI_TEST_CONTENT_MANAGEMENT"
envVarBlobDigest = "OCI_BLOB_DIGEST"
envVarManifestDigest = "OCI_MANIFEST_DIGEST"
envVarTagName = "OCI_TAG_NAME"
envVarTagList = "OCI_TAG_LIST"
testTagName = "tagtest0"
envTrue = "1"
envVarPush = "OCI_TEST_PUSH"
envVarContentDiscovery = "OCI_TEST_CONTENT_DISCOVERY"
envVarContentManagement = "OCI_TEST_CONTENT_MANAGEMENT"
envVarBlobDigest = "OCI_BLOB_DIGEST"
envVarManifestDigest = "OCI_MANIFEST_DIGEST"
envVarTagName = "OCI_TAG_NAME"
envVarTagList = "OCI_TAG_LIST"
envVarHideSkippedWorkflows = "OCI_HIDE_SKIPPED_WORKFLOWS"
testTagName = "tagtest0"

titlePull = "Pull"
titlePush = "Push"
titleContentDiscovery = "Content Discovery"
titleContentManagement = "Content Management"

push = 1 << iota
contentDiscovery
Expand Down

0 comments on commit dca82e7

Please sign in to comment.