diff --git a/conformance/01_pull_test.go b/conformance/01_pull_test.go index 66c32fb9..7c7108e9 100644 --- a/conformance/01_pull_test.go +++ b/conformance/01_pull_test.go @@ -11,7 +11,7 @@ import ( ) var test01Pull = func() { - g.Context("Pull", func() { + g.Context(titlePull, func() { var tag string diff --git a/conformance/02_push_test.go b/conformance/02_push_test.go index b15bc6db..5a02dac8 100644 --- a/conformance/02_push_test.go +++ b/conformance/02_push_test.go @@ -10,7 +10,7 @@ import ( ) var test02Push = func() { - g.Context("Push", func() { + g.Context(titlePush, func() { var lastResponse *reggie.Response diff --git a/conformance/03_discovery_test.go b/conformance/03_discovery_test.go index 08d6ec85..d4dfd512 100644 --- a/conformance/03_discovery_test.go +++ b/conformance/03_discovery_test.go @@ -13,7 +13,7 @@ import ( ) var test03ContentDiscovery = func() { - g.Context("Content Discovery", func() { + g.Context(titleContentDiscovery, func() { var numTags = 4 var tagList []string diff --git a/conformance/04_management_test.go b/conformance/04_management_test.go index 4e3f757b..6f0819e5 100644 --- a/conformance/04_management_test.go +++ b/conformance/04_management_test.go @@ -10,7 +10,7 @@ import ( ) var test04ContentManagement = func() { - g.Context("Content Management", func() { + g.Context(titleContentManagement, func() { const defaultTagName = "tagtest0" var tagToDelete string diff --git a/conformance/README.md b/conformance/README.md index 3162d4cb..0ba5854a 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -108,6 +108,15 @@ OCI_TEST_CONTENT_MANAGEMENT=1 Note: The Content Management tests explicitly depend upon the Push and Content Discovery tests, as there is no way to test content management without also supporting push and content discovery. +#### HTML Report +By default, the HTML report will show tests from all workflows. To hide workflows that have been disabled from +the report, you must set the following in the environment: + +``` +# Required to hide disabled workflows +OCI_HIDE_SKIPPED_WORKFLOWS=1 +``` + #### Container Image You may use the [Dockerfile](./Dockerfile) located in this directory diff --git a/conformance/reporter.go b/conformance/reporter.go index 818bc5a4..736502f1 100644 --- a/conformance/reporter.go +++ b/conformance/reporter.go @@ -176,115 +176,118 @@ const ( } - -

OCI Distribution Conformance Tests

-
-
-
Summary:
-
- {{- if gt .SuiteSummary.NumberOfPassedSpecs 0 -}} - - {{- if .AllPassed -}}All {{ end -}}{{ .SuiteSummary.NumberOfPassedSpecs }} passed - {{- end -}} - {{- if gt .SuiteSummary.NumberOfFailedSpecs 0 -}} - - {{- if .AllFailed -}}All {{ end -}}{{ .SuiteSummary.NumberOfFailedSpecs }} failed - {{- end -}} - {{- if gt .SuiteSummary.NumberOfSkippedSpecs 0 -}} - - {{- if .AllSkipped -}}All {{ end -}}{{ .SuiteSummary.NumberOfSkippedSpecs }} skipped - {{- end -}} -
-
-
+ +

OCI Distribution Conformance Tests

+
+
+
Summary:
+
+ {{- if gt .SuiteSummary.NumberOfPassedSpecs 0 -}} + + {{- if .AllPassed -}}All {{ end -}}{{ .SuiteSummary.NumberOfPassedSpecs }} passed + {{- end -}} + {{- if gt .SuiteSummary.NumberOfFailedSpecs 0 -}} + + {{- if .AllFailed -}}All {{ end -}}{{ .SuiteSummary.NumberOfFailedSpecs }} failed + {{- end -}} + {{- if gt .SuiteSummary.NumberOfSkippedSpecs 0 -}} + + {{- if .AllSkipped -}}All {{ end -}}{{ .SuiteSummary.NumberOfSkippedSpecs }} skipped + {{- end -}} +
+
+
+
+
+
-
-
-
Start time:
-
{{ .StartTimeString }}
-
-
-
End time:
-
{{ .EndTimeString }}
-
-
-
Elapsed time:
-
{{ .RunTime }}
-
-
-
Test version:
-
{{ .Version }}
-
-
-
Configuration:
-
- {{ range $i, $s := .EnvironmentVariables }} - {{ $s }}
- {{ end }} +
+
Start time:
+
{{ .StartTimeString }}
+
+
+
End time:
+
{{ .EndTimeString }}
+
+
+
Elapsed time:
+
{{ .RunTime }}
+
+
+
Test version:
+
{{ .Version }}
+
+
+
Configuration:
+
+ {{ range $i, $s := .EnvironmentVariables }} + {{ $s }}
+ {{ end }} +
-
-
- {{with .Suite}} - {{$suite := .M}} - {{range $i, $suiteKey := .Keys}} -

{{$suiteKey}}

-
- {{$wf := index $suite $suiteKey}} - {{with $wf}} - {{$workflow := .M}} - {{range $j, $workflowKey := .Keys}} -

{{$workflowKey}}

- {{$ctg := index $workflow $workflowKey}} - {{with $ctg}} - {{$category := .M}} - {{range $k, $categoryKey := .Keys}} - {{$s := index $category $categoryKey}} - {{if eq $s.State 4}} -
-
+
-

{{$s.Title}}

-
-
- -
- 
{{$s.Failure.Message}}
-
-
- {{else if eq $s.State 3}} -
-
+
-

{{$s.Title}}

-
- -
- {{else if eq $s.State 2}} -
-
+
-

{{$s.Title}}

-
- -
- {{end}} - {{end}}
- {{end}} - {{end}} - {{end}} -
- {{end}} - {{end}} +
+ {{with .Suite}} + {{$suite := .M}} + {{range $i, $suiteKey := .Keys}} + {{$wf := index $suite $suiteKey}} + {{with $wf}} + {{ if .IsEnabled }} +

{{$suiteKey}}

+
+ {{$workflow := .M}} + {{range $j, $workflowKey := .Keys}} +

{{$workflowKey}}

+ {{$ctg := index $workflow $workflowKey}} + {{with $ctg}} + {{$category := .M}} + {{range $k, $categoryKey := .Keys}} + {{$s := index $category $categoryKey}} + {{if eq $s.State 4}} +
+
+
+

{{$s.Title}}

+
+
+ +
+ 
{{$s.Failure.Message}}
+
+
+ {{else if eq $s.State 3}} +
+
+
+

{{$s.Title}}

+
+ +
+ {{else if eq $s.State 2}} +
+
+
+

{{$s.Title}}

+
+ +
+ {{end}} + {{end}}
+ {{end}} + {{end}} + {{end}} + {{end}} +
+ {{end}} + {{end}}
- - + ` ) @@ -302,8 +305,9 @@ type ( } workflow struct { - M map[string]*category - Keys []string + M map[string]*category + IsEnabled bool + Keys []string } category struct { @@ -340,6 +344,7 @@ type ( SuiteSummary *types.SuiteSummary debugLogger *httpDebugWriter debugIndex int + enabledMap map[string]bool PercentPassed int PercentFailed int PercentSkipped int @@ -431,10 +436,27 @@ func (l *httpDebugLogger) output(format string, v ...interface{}) { } } -func newHTMLReporter(htmlReportFilename string) *HTMLReporter { +func newHTMLReporter(htmlReportFilename string) (h *HTMLReporter) { + enabledMap := map[string]bool{ + titlePull: true, + titlePush: true, + titleContentDiscovery: true, + titleContentManagement: true, + } + + if os.Getenv(envVarHideSkippedWorkflows) == "1" { + enabledMap = map[string]bool{ + titlePull: true, + titlePush: !userDisabled(push), + titleContentDiscovery: !userDisabled(contentDiscovery), + titleContentManagement: !userDisabled(contentManagement), + } + } + return &HTMLReporter{ htmlReportFilename: htmlReportFilename, debugLogger: httpWriter, + enabledMap: enabledMap, SpecSummaryMap: summaryMap{M: make(map[string]snapShotList)}, Suite: suite{ M: make(map[string]*workflow), @@ -450,19 +472,19 @@ func (reporter *HTMLReporter) SpecDidComplete(specSummary *types.SpecSummary) { } specSummary.CapturedOutput = b.String() - //header := specSummary.ComponentTexts[categoryIndex] snapshot := newSpecSnapshot(specSummary, reporter.Suite.Size) - reporter.Save(snapshot) + reporter.save(snapshot) reporter.debugIndex = len(reporter.debugLogger.CapturedOutput) } -func (reporter *HTMLReporter) Save(snapshot *specSnapshot) { +func (reporter *HTMLReporter) save(snapshot *specSnapshot) { suite := &reporter.Suite ct := snapshot.ComponentTexts suiteName, categoryName, specTitle := ct[flowIndex], ct[categoryIndex], ct[specIndex] //make the map of categories if _, ok := suite.M[suiteName]; !ok { - suite.M[suiteName] = &workflow{M: make(map[string]*category), Keys: []string{}} + suite.M[suiteName] = &workflow{M: make(map[string]*category), Keys: []string{}, + IsEnabled: reporter.enabledMap[suiteName]} suite.Keys = append(suite.Keys, suiteName) } //make the map of snapshots diff --git a/conformance/setup.go b/conformance/setup.go index 6c15bade..0b19601e 100644 --- a/conformance/setup.go +++ b/conformance/setup.go @@ -36,15 +36,21 @@ const ( DENIED UNSUPPORTED - envTrue = "1" - envVarPush = "OCI_TEST_PUSH" - envVarContentDiscovery = "OCI_TEST_CONTENT_DISCOVERY" - envVarContentManagement = "OCI_TEST_CONTENT_MANAGEMENT" - envVarBlobDigest = "OCI_BLOB_DIGEST" - envVarManifestDigest = "OCI_MANIFEST_DIGEST" - envVarTagName = "OCI_TAG_NAME" - envVarTagList = "OCI_TAG_LIST" - testTagName = "tagtest0" + envTrue = "1" + envVarPush = "OCI_TEST_PUSH" + envVarContentDiscovery = "OCI_TEST_CONTENT_DISCOVERY" + envVarContentManagement = "OCI_TEST_CONTENT_MANAGEMENT" + envVarBlobDigest = "OCI_BLOB_DIGEST" + envVarManifestDigest = "OCI_MANIFEST_DIGEST" + envVarTagName = "OCI_TAG_NAME" + envVarTagList = "OCI_TAG_LIST" + envVarHideSkippedWorkflows = "OCI_HIDE_SKIPPED_WORKFLOWS" + testTagName = "tagtest0" + + titlePull = "Pull" + titlePush = "Push" + titleContentDiscovery = "Content Discovery" + titleContentManagement = "Content Management" push = 1 << iota contentDiscovery