From 89612da14acb7f824fd19acd9101c82af6bfdf7f Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Tue, 14 Jul 2020 21:20:40 -0400 Subject: [PATCH 1/2] scorecard: use TestList for output, exit 1 on test failure --- changelog/fragments/scorecard-output-api.yaml | 15 ++++ cmd/operator-sdk/alpha/scorecard/cmd.go | 43 +++++---- internal/scorecard/alpha/config.go | 12 +-- .../bundle/tests/scorecard/config.yaml | 12 +-- internal/scorecard/alpha/formatting.go | 38 ++++---- internal/scorecard/alpha/formatting_test.go | 17 +--- internal/scorecard/alpha/labels_test.go | 30 ++----- internal/scorecard/alpha/run_test.go | 11 +-- internal/scorecard/alpha/scorecard.go | 67 +++++++++----- .../scorecard/alpha/testdata/bundle.tar.gz | Bin 2865 -> 2840 bytes .../bundle/tests/scorecard/config.yaml | 24 ++--- internal/scorecard/alpha/testpod.go | 9 +- .../v1alpha2/zz_generated.deepcopy.go | 25 +++--- pkg/apis/scorecard/v1alpha3/formatter.go | 85 ++++++++++-------- pkg/apis/scorecard/v1alpha3/types.go | 29 ++++-- .../v1alpha3/zz_generated.deepcopy.go | 55 ++++++++---- .../content/en/docs/scorecard/custom-tests.md | 6 +- .../content/en/docs/scorecard/kuttl-tests.md | 4 +- .../en/docs/scorecard/scorecard-alpha.md | 73 +++++++++------ 19 files changed, 307 insertions(+), 248 deletions(-) create mode 100644 changelog/fragments/scorecard-output-api.yaml diff --git a/changelog/fragments/scorecard-output-api.yaml b/changelog/fragments/scorecard-output-api.yaml new file mode 100644 index 0000000000..482c8ba08d --- /dev/null +++ b/changelog/fragments/scorecard-output-api.yaml @@ -0,0 +1,15 @@ +# entries is a list of entries to include in +# release notes and/or the migration guide +entries: + - description: > + Changed scorecard text and json output to use a `v1alpha3.TestList` + instead of aggregating all test results under a single + `v1alpha3.Test` and set exit status to 1 when a test fails. + kind: "change" + breaking: true + migration: + header: Alpha scorecard output API updates + body: | + Update any scripts interpretting the scorecard output to + understand the v1alpha3 TestList format. + diff --git a/cmd/operator-sdk/alpha/scorecard/cmd.go b/cmd/operator-sdk/alpha/scorecard/cmd.go index 6d58719417..8a7ba4d9ad 100644 --- a/cmd/operator-sdk/alpha/scorecard/cmd.go +++ b/cmd/operator-sdk/alpha/scorecard/cmd.go @@ -86,24 +86,20 @@ If the argument holds an image tag, it must be present remotely.`, return scorecardCmd } -func (c *scorecardCmd) printOutput(output v1alpha3.Test) error { +func (c *scorecardCmd) printOutput(output v1alpha3.TestList) error { switch c.outputFormat { case "text": - if len(output.Status.Results) == 0 { + if len(output.Items) == 0 { fmt.Println("0 tests selected") return nil } - o, err := output.MarshalText() - if err != nil { - fmt.Println(err.Error()) - return err + for _, test := range output.Items { + fmt.Println(test.MarshalText()) } - fmt.Printf("%s\n", o) case "json": bytes, err := json.MarshalIndent(output, "", " ") if err != nil { - fmt.Println(err.Error()) - return err + return fmt.Errorf("marshal json error: %v", err) } fmt.Printf("%s\n", string(bytes)) default: @@ -154,12 +150,9 @@ func (c *scorecardCmd) run() (err error) { return fmt.Errorf("could not parse selector %w", err) } - var scorecardTest v1alpha3.Test + var scorecardTests v1alpha3.TestList if c.list { - scorecardTest, err = o.ListTests() - if err != nil { - return fmt.Errorf("error listing tests %w", err) - } + scorecardTests = o.List() } else { runner := scorecard.PodTestRunner{ ServiceAccount: c.serviceAccount, @@ -178,13 +171,31 @@ func (c *scorecardCmd) run() (err error) { ctx, cancel := context.WithTimeout(context.Background(), c.waitTime) defer cancel() - scorecardTest, err = o.RunTests(ctx) + scorecardTests, err = o.Run(ctx) if err != nil { return fmt.Errorf("error running tests %w", err) } } - return c.printOutput(scorecardTest) + if err := c.printOutput(scorecardTests); err != nil { + log.Fatal(err) + } + + if hasFailingTest(scorecardTests) { + os.Exit(1) + } + return nil +} + +func hasFailingTest(list v1alpha3.TestList) bool { + for _, t := range list.Items { + for _, r := range t.Status.Results { + if r.State != v1alpha3.PassState { + return true + } + } + } + return false } func (c *scorecardCmd) validate(args []string) error { diff --git a/internal/scorecard/alpha/config.go b/internal/scorecard/alpha/config.go index 254de68be7..f05f8016cd 100644 --- a/internal/scorecard/alpha/config.go +++ b/internal/scorecard/alpha/config.go @@ -26,12 +26,12 @@ const ( ) type Test struct { - Name string `yaml:"name"` // The container test name - Image string `yaml:"image"` // The container image name - // An list of commands and arguments passed to the test image - Entrypoint []string `yaml:"entrypoint,omitempty"` - Labels map[string]string `yaml:"labels"` // User defined labels used to filter tests - Description string `yaml:"description"` // User readable test description + // Image is the name of the testimage + Image string `json:"image"` + // Entrypoint is list of commands and arguments passed to the test image + Entrypoint []string `json:"entrypoint,omitempty"` + // Labels that further describe the test and enable selection + Labels map[string]string `json:"labels,omitempty"` } // Config represents the set of test configurations which scorecard diff --git a/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml b/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml index f5fbe5da9c..8ccedd19e4 100644 --- a/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/alpha/examples/custom-scorecard-tests/bundle/tests/scorecard/config.yaml @@ -1,19 +1,15 @@ tests: -- name: "customtest1" - image: quay.io/username/custom-scorecard-tests:dev - entrypoint: +- image: quay.io/username/custom-scorecard-tests:dev + entrypoint: - custom-scorecard-tests - customtest1 labels: suite: custom test: customtest1 - description: an ISV custom test -- name: "customtest2" - entrypoint: +- image: quay.io/username/custom-scorecard-tests:dev + entrypoint: - custom-scorecard-tests - customtest2 - image: quay.io/username/custom-scorecard-tests:dev labels: suite: custom test: customtest2 - description: an ISV custom test \ No newline at end of file diff --git a/internal/scorecard/alpha/formatting.go b/internal/scorecard/alpha/formatting.go index 67c2eca737..15d0fc18f8 100644 --- a/internal/scorecard/alpha/formatting.go +++ b/internal/scorecard/alpha/formatting.go @@ -18,47 +18,41 @@ import ( "context" "encoding/json" - "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" v1 "k8s.io/api/core/v1" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) // getTestResult fetches the test pod log and converts it into // Test format -func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod, test Test) (output *v1alpha3.TestStatus) { - +func (r PodTestRunner) getTestStatus(ctx context.Context, p *v1.Pod) (output *v1alpha3.TestStatus) { logBytes, err := getPodLog(ctx, r.Client, p) if err != nil { - return testStatusError(err, test) + return convertErrorToStatus(err, string(logBytes)) } // marshal pod log into TestResult err = json.Unmarshal(logBytes, &output) if err != nil { - return testStatusError(err, test) + return convertErrorToStatus(err, string(logBytes)) } return output } -// ListTests lists the scorecard tests as configured that would be +// List lists the scorecard tests as configured that would be // run based on user selection -func (o Scorecard) ListTests() (output v1alpha3.Test, err error) { +func (o Scorecard) List() v1alpha3.TestList { + output := v1alpha3.NewTestList() tests := o.selectTests() - if len(tests) == 0 { - return output, err - } for _, test := range tests { - output.Status.Results = append(output.Status.Results, v1alpha3.TestResult{Name: test.Name}) + item := v1alpha3.NewTest() + item.Spec = v1alpha3.TestSpec{ + Image: test.Image, + Entrypoint: test.Entrypoint, + Labels: test.Labels, + } + output.Items = append(output.Items, item) } - return output, err -} - -func testStatusError(err error, test Test) *v1alpha3.TestStatus { - r := v1alpha3.TestResult{} - r.Name = test.Name - r.State = v1alpha3.FailState - r.Errors = []string{err.Error()} - return &v1alpha3.TestStatus{ - Results: []v1alpha3.TestResult{r}, - } + return output } diff --git a/internal/scorecard/alpha/formatting_test.go b/internal/scorecard/alpha/formatting_test.go index 9a91f4d93b..02b482987e 100644 --- a/internal/scorecard/alpha/formatting_test.go +++ b/internal/scorecard/alpha/formatting_test.go @@ -18,7 +18,6 @@ import ( "path/filepath" "testing" - "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" "k8s.io/apimachinery/pkg/labels" ) @@ -27,10 +26,9 @@ func TestList(t *testing.T) { cases := []struct { bundlePathValue string selector string - wantError bool resultCount int }{ - {"testdata/bundle", "suite=basic", false, 1}, + {"testdata/bundle", "suite=basic", 1}, } for _, c := range cases { @@ -50,17 +48,8 @@ func TestList(t *testing.T) { } runner.BundlePath = c.bundlePathValue o.TestRunner = &runner - var output v1alpha3.Test - output, err = o.ListTests() - if err == nil && c.wantError { - t.Fatalf("Wanted error but got no error") - } else if err != nil { - if !c.wantError { - t.Fatalf("Wanted result but got error: %v", err) - } - return - } - actualResultCount := len(output.Status.Results) + output := o.List() + actualResultCount := len(output.Items) if c.resultCount != actualResultCount { t.Fatalf("Wanted result count %d but got : %d", c.resultCount, actualResultCount) } diff --git a/internal/scorecard/alpha/labels_test.go b/internal/scorecard/alpha/labels_test.go index eace28bf57..1959f98418 100644 --- a/internal/scorecard/alpha/labels_test.go +++ b/internal/scorecard/alpha/labels_test.go @@ -66,64 +66,50 @@ func TestEmptySelector(t *testing.T) { } const testConfig = `tests: -- name: "customtest1" - image: quay.io/someuser/customtest1:v0.0.1 +- image: quay.io/someuser/customtest1:v0.0.1 entrypoint: - custom-test labels: suite: custom test: customtest1 - description: an ISV custom test that does... -- name: "customtest2" - image: quay.io/someuser/customtest2:v0.0.1 +- image: quay.io/someuser/customtest2:v0.0.1 entrypoint: - custom-test labels: suite: custom test: customtest2 - description: an ISV custom test that does... -- name: "basic-check-spec" - image: quay.io/redhat/basictests:v0.0.1 +- image: quay.io/redhat/basictests:v0.0.1 entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - description: check the spec test -- name: "basic-check-status" - image: quay.io/redhat/basictests:v0.0.1 +- image: quay.io/redhat/basictests:v0.0.1 entrypoint: - scorecard-test - basic-check-status labels: suite: basic test: basic-check-status-test - description: check the status test -- name: "olm-bundle-validation" - image: quay.io/redhat/olmtests:v0.0.1 +- image: quay.io/redhat/olmtests:v0.0.1 entrypoint: - scorecard-test - olm-bundle-validation labels: suite: olm test: olm-bundle-validation-test - description: validate the bundle test -- name: "olm-crds-have-validation" - image: quay.io/redhat/olmtests:v0.0.1 +- image: quay.io/redhat/olmtests:v0.0.1 entrypoint: - scorecard-test - olm-crds-have-validation labels: suite: olm test: olm-crds-have-validation-test - description: CRDs have validation -- name: "kuttl-tests" - image: quay.io/redhat/kuttltests:v0.0.1 +- image: quay.io/redhat/kuttltests:v0.0.1 labels: suite: kuttl entrypoint: - kuttl-test - olm-status-descriptors - description: Kuttl tests -` + ` diff --git a/internal/scorecard/alpha/run_test.go b/internal/scorecard/alpha/run_test.go index 84e5928d0f..f18eaa4c43 100644 --- a/internal/scorecard/alpha/run_test.go +++ b/internal/scorecard/alpha/run_test.go @@ -20,8 +20,9 @@ import ( "testing" "time" - "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" "k8s.io/apimachinery/pkg/labels" + + "github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3" ) func TestRunTests(t *testing.T) { @@ -79,11 +80,11 @@ func TestRunTests(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Duration(7*time.Second)) defer cancel() - var scorecardOutput v1alpha3.Test - scorecardOutput, err = o.RunTests(ctx) + var scorecardOutput v1alpha3.TestList + scorecardOutput, err = o.Run(ctx) - if scorecardOutput.Status.Results[0].State != c.expectedState { - t.Fatalf("Wanted state %v, got %v", c.expectedState, scorecardOutput.Status.Results[0].State) + if scorecardOutput.Items[0].Status.Results[0].State != c.expectedState { + t.Fatalf("Wanted state %v, got %v", c.expectedState, scorecardOutput.Items[0].Status.Results[0].State) } if err == nil && c.wantError { diff --git a/internal/scorecard/alpha/scorecard.go b/internal/scorecard/alpha/scorecard.go index 2813fa2282..03fbb4ff96 100644 --- a/internal/scorecard/alpha/scorecard.go +++ b/internal/scorecard/alpha/scorecard.go @@ -57,11 +57,11 @@ type FakeTestRunner struct { Error error } -// RunTests executes the scorecard tests as configured -func (o Scorecard) RunTests(ctx context.Context) (testOutput v1alpha3.Test, err error) { +// Run executes the scorecard tests as configured +func (o Scorecard) Run(ctx context.Context) (v1alpha3.TestList, error) { + testOutput := v1alpha3.NewTestList() - err = o.TestRunner.Initialize(ctx) - if err != nil { + if err := o.TestRunner.Initialize(ctx); err != nil { return testOutput, err } @@ -73,14 +73,21 @@ func (o Scorecard) RunTests(ctx context.Context) (testOutput v1alpha3.Test, err for _, test := range tests { result, err := o.TestRunner.RunTest(ctx, test) if err != nil { - result = convertErrorToStatus(test.Name, err) + result = convertErrorToStatus(err, "") + } + + out := v1alpha3.NewTest() + out.Spec = v1alpha3.TestSpec{ + Image: test.Image, + Entrypoint: test.Entrypoint, + Labels: test.Labels, } - testOutput.Status.Results = append(testOutput.Status.Results, result.Results...) + out.Status = *result + testOutput.Items = append(testOutput.Items, out) } if !o.SkipCleanup { - err = o.TestRunner.Cleanup(ctx) - if err != nil { + if err := o.TestRunner.Cleanup(ctx); err != nil { return testOutput, err } } @@ -102,8 +109,13 @@ func (o Scorecard) selectTests() []Test { return selected } -func (r FakeTestRunner) Initialize(ctx context.Context) (err error) { - return nil +func (r FakeTestRunner) Initialize(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } } // Initialize sets up the bundle configmap for tests @@ -121,8 +133,13 @@ func (r *PodTestRunner) Initialize(ctx context.Context) error { } -func (r FakeTestRunner) Cleanup(ctx context.Context) (err error) { - return nil +func (r FakeTestRunner) Cleanup(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } } // Cleanup deletes pods and configmap resources from this test run @@ -139,27 +156,30 @@ func (r PodTestRunner) Cleanup(ctx context.Context) (err error) { } // RunTest executes a single test -func (r PodTestRunner) RunTest(ctx context.Context, test Test) (result *v1alpha3.TestStatus, err error) { - +func (r PodTestRunner) RunTest(ctx context.Context, test Test) (*v1alpha3.TestStatus, error) { // Create a Pod to run the test podDef := getPodDefinition(r.configMapName, test, r) pod, err := r.Client.CoreV1().Pods(r.Namespace).Create(ctx, podDef, metav1.CreateOptions{}) if err != nil { - return result, err + return nil, err } err = r.waitForTestToComplete(ctx, pod) if err != nil { - return result, err + return nil, err } - result = r.getTestStatus(ctx, pod, test) - return result, nil + return r.getTestStatus(ctx, pod), nil } // RunTest executes a single test func (r FakeTestRunner) RunTest(ctx context.Context, test Test) (result *v1alpha3.TestStatus, err error) { - return r.TestStatus, r.Error + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + return r.TestStatus, r.Error + } } func ConfigDocLink() string { @@ -182,17 +202,16 @@ func (r PodTestRunner) waitForTestToComplete(ctx context.Context, p *v1.Pod) (er return false, nil }) - err = wait.PollImmediateUntil(time.Duration(1*time.Second), podCheck, ctx.Done()) + err = wait.PollImmediateUntil(1*time.Second, podCheck, ctx.Done()) return err } -func convertErrorToStatus(name string, err error) *v1alpha3.TestStatus { +func convertErrorToStatus(err error, log string) *v1alpha3.TestStatus { result := v1alpha3.TestResult{} - result.Name = name - result.Errors = []string{err.Error()} - result.Suggestions = []string{} result.State = v1alpha3.FailState + result.Errors = []string{err.Error()} + result.Log = log return &v1alpha3.TestStatus{ Results: []v1alpha3.TestResult{result}, } diff --git a/internal/scorecard/alpha/testdata/bundle.tar.gz b/internal/scorecard/alpha/testdata/bundle.tar.gz index 4f569ab3c15911047bcb8a161fa7a26c01a8a3e8..f79410dadcf527f29bac3900d3a00a8ecc8fc32c 100644 GIT binary patch literal 2840 zcmV+z3+MD7iwFP!00000|Li~8c;2rZ9-Mb*Jb#e<^BV!H{|;u1E361- z(%s;gzUUrW9>acrul|Rl28ClTUbT>llt2Q7qn0s<>8q?`Y5E^rJVxmdY` zyR(!<6xDJE`!D!ec485zKK)l_8XoIEnrX(ZAvzN5}u3 zKj>>0^1J%u9RE)MrRy0VaN!e<6K$6~{!loE{1A%h>3QKm{2%zeVWIz?wv(&=z2D^o4UQ72{#D1nB~qEOgr)-1 zDhmcJ30tHTi^@O?AsO-n0(7glOK@m$=}ctLn6V%gR;dio?_XL7tuZB$Nt$+$C&0#& zo1c2qa0xNCNQ5Ggp&*HwS8|DnL~04h&P8)B%-51vG@jgK6cV9xhax!_8Y&tqaE-k# zH8XY&inNZvZn?HNh>*0K^EpyWB9VYZfCOeMfeKlqr(lNo>{keti>4qn5hRy9r4b4_ zTZ2$YK*$%2d@d+Uq`29VVg;F2av)5BV3=w#rtlrYEJ88fAOU(0l(GsnGex@S9T$KI zoAUrIl}cpL=`4uKZ8YR@oHC*|oscsn$Sl=)w-dn{Xh)KTgGIO`3PP0%=-`AnhH+-= z-8lNcLfMxUPIZB!ld&LdQJ{?}?v=YXNUdL-EwlWW;?*cEmAmWdJ<2yVDyYlcNV{An zOTlYF5jF>56*m_=HWYzH2`5aMFCits>Oz<4EG9~8`nObvd!)FFu1xiZW`NQp!U`f6 z-7s{8W1v@9!mEp2OISKuq=yAx4KVeIR$2u*4=p=2RCwVufw1Nq5lAAm{VJ?LTYWv; z?Dt0-Sw2G@)3l76+v)Y)JsQ7xb8~<5{&xHpO()k^=!cu@w{ND%+1@^+sZ<~uL=n1) z6D|~HDnNHBL+!k{bY!$b3t;*ipw;)XtzA$+feLv92+w!XjCn-n8*7}}s0al#jv^GA zXkQ6;DAh3(tHkB$m!nQ^{y*1oGh-O4g;J~P;G7JZmuNkZAl_#|v$ zijAd;=@!)~`3!mZ)FQX>s}eg=1PSGv7?`?jte$fbV-@J=)9W_ItR3mNw5Ul(JB!vz z_4ypL^FbrD>Ef^%nt3%t3QbuW`PqxV| znx&0C$0oPP8XJ8jo4nNmM_7PSo+(GYz^-G*#)5+?($}mFKLIR{j3R#`R1ztdg`P$X(84f|&|7{` zV|aWY_;CN{d%fPM(f{=Z!{NFAI|Wqxzw*Q?5Nhj~ zrWa;gIE5FqEAzh8i0J}zn~<&w+5zhAUuIJM4i0prZU=~^w3=PMl>aTTU$X!GVm?Eh z;2ELB6anZTI%T07CKWbx8clfV)N9Z6Tt9EHb;lc^gyImE+$7Jue#QyO96u`B{Ja6S zNjPs{s^hjF{>butXgAw+?Ci+)+oHmnB$^I#SM<{E4=|r;~5B@-pkZZ zw8U8mrsK8cr&}m2A~KN%{gOJCr?yNW? zp_`H*;n5XO%d)p|e4)u$F-~|{Mph(~%fuMzVW}Kl_3ReKZLK=7vZz-q&k}|qb%|;o zRAYZq+Iw}N)nK!<*7xO`0TSAi7YEI-gkj&rJ=C`GqIffsVTkp|>

WnU;-8Ma27j zc5iPhAfY70GC+O>D}icZhYN}PbcH6 z>#cdnp8NbDc)U$S!<+z(?qI%qG#j57w6~!eRlcIJcCSzwBjr^8Srs z_cWa9J@>NmesX;`zJGuBAU_9A(_#kUWWOhCj$$}($e8HtoaE!^RR{3v*u$a;I%Znt)^ zRGU)^zvke=vbP)AnG=R7`3#~yo*yjMv5C$cQUOzIPSY4dib>p9xQ6Dckh3{i#5ifP z5DKVGU4xi4o+cDfKQYi5i9!Rl09E@=iL4_M9R+I`d-PdDeXLE|s|1|X9%PRiDv`d9 z#}IQSp@GgrdJ}IBVdg27-=d^q&WPfIutlY{I4ru%HnY_S9=rKK4KX}ci?`+Lr{XCd ziVlahzuUuhJV`^&FBt?5$nBY?zRz}dyTDU<*;nU3z}V&gU;a^j<`FrgtieiwmNn+P z{N&d`GEEXLRPA8S7Nze$vZ7!(Fm+1Ug2LX}b4+D*`n!}WaxCpJrDfgHLTCk>k6c6< z`^K(gOsch=W3z3HRVs+uWVdZv<{*4cA`tnj*}SE(UGCRDUirgAUbMaD9*?^JGaOxb zez)uO&-Z^$0maq1SG)mw^!-n*e~tGC{n7dS_atz>|M}Ox|EXU2?&rnP@qaiR_WRwT z*Sqk1e=r=Ke0es9n_=l_#HL;oFnMDwy0z=8bV z^#-Gc{s(&aoc~V%M*l&}$xl;%OYz1fyi;7^DUw!cB#vGbM6hll1WbvIIUo#>ZTf1n z^7kYpbiyTsD@P^}w(4&JWJ(kSc9!iCHTkVomfou}%4zJ_yzG?si*|F%X) z9EwOfOT2yx&%4&1obBSw=J8w?OHahKvBS!X$zwTgt~(XG=56ki7~WcGDSz#7b%qy) zX8vELO1$S8D)fHz-a5|xw zLH>T=Yd_u@4!~~zr$j0z;nB;S3_U*n4~M>gj{m2C1N!fHo`aj0|LBhz`tN(*spJ21>3pO5-}<%Qv#%{Yn#j)E q6EAR)%NJ0RR64*y#EIRsaCAk(H_d literal 2865 zcmV-13(oW(iwFP!00000|Li+mbK*F*dFEHBxDTnicWi73FxPL~bY^a9I|&uCvky~K zUBW(qHnQZAWV-3u{_k6o{EcbI7d<`8c?jWHIy&DvM;0+=WC2piv$q4!^StqRs6YMj z(5rstaOMw2pl~Vu%egk{?15xf>kQXRV?2F&bU$vj5N@jOzC95B#xzhK{1d|8V=` zg!}?R63zmI69S(UF#S!sk3UM6@ZQ>=gTj92k+3L0m#I`dz6Z%u5yCrI5JnW?tP_L6 z5mq?pAjEJC0V=)6-EystOd!-lmx8BBfVKx%DpHR@$gX{p6ErwVr21PO|DH%?o+UID zm{wUZXi3;IrC3xRbP$puPar_IdbtFL7MIRNCS%5eR9K}lK!1GcAhgDmL|DshAMym) zWP1I}U>2?*#ukZC1TqvPG2==u5s^r(0NJrx@DocmhI^oC^&Vixs%W zTIZS>I|fDCMqrm*TO344+Qs<-sTGk(Kq5c_Q%0x?S)_+xhWPxS5GohVKxiUJu6Rl# z6mqr(p^$)(FB$nvn0g|GOOf3m;%8t)oM)P2ZULKV!S~D^cpB-6>4UPbkPkL zfCyXg0IiftWYFs^iOPL6nA`@F)b+GxL!N)Waz(8d*a>RmgewlB`6S^i7)YLwQ>{nhLN<%=2<)OBvO zT~3pw;N-iol|T6Q<0UkP={ZrOR|46QwQvd#cktQrtzCm@%%<98j7> zSV82XYlbdy4D=F9cyqOD4NFI>bid-O38p^KYO6r!sioCIMHJ2w2y3wsfh0ovufhtn z*Vog{?s&ABX$U_VOW3r2^3) ziqLhOaG@|$0lH5a>S~6t;=+;9BU%E}-vB-SD7!iY1r(@|M}Y96jb_Xvve;PX)J{bx zph*;=&}92cxCvMD+$yk*Y$qqFv1{pnVwY4@4KhO_9y97__) zUdAV3OH*vDR7|(1&d6uT)2CLs&0m$=i6TfS-^9Swd1Lm1ix{gwXP-g8@yz;>&P&Uh zc66|4t5oH4%*uP6&~6v|-O!AyIU=7+9qZ5ZM8=~g>Ht@Km^}P_W}VUaqUNT-$d)lm zAAPn@ZqY1#^d&yIMb`M}8~Nm|Rye`}jPXp-vh9`$uV*+7L_ypVIjzaB`UnewH+!Ab z_`%n`w1HaZW#HP!G+AN4#2Gf7tgfp(M#y?dfRsoftCP6EOUN+ay1G5VJ}CRY!dsm@ zeBCU*wUJ2(nKL}S5Bz%n=X-;}xY7R&{r=$8{~ZIW{a4sbY*B^1O2=}Z3v@S8T zE|E>Gb$_O&G?_WeZkWbS*4oJc{maSH&#y&x`lFJkx+O#10Cl$-!-Ajg57k+kIX%AE zD(jDP{<=YJF@(B0r|F5=5>62X-O9KxHDbEJ+$N-}f^L8YyQi70zJ&vwsk;GUDXs1< zU&?0*t+8lP(pDCD{eN= zyneTN z{7~r)SQ^hk=ZRl-C)(yl^K+gG z7GN3&~=ErM$7PfLHVKIk;qEWPzEi=GZbB(x_l4w_>LqYIPwP}k;*7$Smg zVP!DH#$$GdROw91Mx~!vILNWf;gdAHJabHUR?uiUHzc-w`E7BUtkIo24ah-gF+t1xPDlZKiadB24L&9 znU>RkOdc-(|HJL%=4v*bTwZOBL-yR~C&A-wA{rJ1Xmk&Y_NUqNsR@f*mu_WW{q>yF z*i~38K^5ng^8B}*3{UUg8Fr7usa|s@D|geY`^m%I{a5+f>kJx5Y#NRGtLgj6<<*Dl zo5|1H1kIpnO6inS5^e%Cp&#)^KF2 zDb&&(PRP%Kr-`h$cinEMjiuU{TKY8y7nZ$tWN$$jrsOk-`t$r?v5rmj&LI^rwdOR9 zA*7hZjfrb$&I&nOkY$XMCJUi}+R!zKc|)2|K>fr(??@CHs3oY{cS>X(k?1T~!`P$G z8tP+h&`u-Zq$bD?GgLBt9giXAOhN;lr}QS?9KtM8D!)ff#hekv1!2odYq4K-nQvyZ z_lVsTsG)|ZTJg4g{ZvTtwd}Cp`t1?cB1uEfubBk)=!r1wL zpZ{57<_UX7S%Z}dEo;o1{N&d`GEEXLRPA8S7NvI|Sy3?T89F0uNnz*c1*Wn({3fM} z97=maX<4^)5PF2oM=qj_edE_LCe_-`vAb=IRVs+u1jw{<9di)ACJ~7I)ok9<*e>^5 zAFurBAurnAa*b!){}~R3r~5z0fZYDQN_+AAn?CKZ=l+j38VnltAA5uG>HPN?&^WCv zkAU<476Qlck&8!{aCe@vh=N<3a=2?hKmwLTDzW(=bBc5zh5WUHp-{ercmfMqvuWn% zPJHn61c#4!3EAwH-S#jt;R-Vb)JkD?q#vZha|%zT;6|{(DOHA0M&c;;S8sgx^Xz%~ z-}lF-{C^BE_U}pEm4#SDZ?eW=_8)uxU^E=~gEP+`dPDDI|6@SI{ylp{^STwlp8dZ+ z_U-;Z?)$@wabNHM{qbr4KMEN82c7(Mwrg&_IN=JyN5{PA>=XyBzo&TP65cCr_Z0hC z=|7I97euh`AOuW_jX66EknIO+qw-fMwdFeXmjp5;3IaRIwvU?p)~uXf?H!MKY_EzX z5Sp2#n4HFr-GQC*Cea(q+PY@Dz?{Yvaa$+0%A7s6Z{_Ti7QsUiNoR%EN0z;9*+a!| zo7<{^%lmf{>4B9SV!Na-4vUT|e6i}&WiO`g5xrcZCb4O8)%(U)ld3Q>9+B_dk19{Wt3SgH!!? z3~05#{cC*hH7db<=Rdx=|5vep-#^uVM}q48zsEG2Cea|2Gr~H2m z*kk{m=Q&tP5HHyX_U8XlGynHT{^*qdj{(*Ezs@uMlt;BI-PPh1CrNZ9lQT~+86#Ta6tSg`~|=LKRY1)``)k_|K8X;#s4wjp!l}|o$$!< zPubwF1=g6oH(kTjf1bC${=4u-V}E$&`QC8QKXU!|Qaayh{Ac!m!`mNu{vYkW|8X&H z*#AOn@VNQ^rF6bi`)~bP@5R>^o=l|m_QWe(BzcY7EHA!}@=6y{zO|d2-~=c53&Otx P00960eqT0=08jt`>X5_F diff --git a/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml b/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml index b56656a08c..265eac4cb7 100644 --- a/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml +++ b/internal/scorecard/alpha/testdata/bundle/tests/scorecard/config.yaml @@ -1,55 +1,43 @@ tests: -- name: "basic-check-spec" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - description: check the spec test -- name: "olm-bundle-validation" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - olm-bundle-validation labels: suite: olm test: olm-bundle-validation-test - description: validate the bundle test -- name: "olm-crds-have-validation" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - olm-crds-have-validation labels: suite: olm test: olm-crds-have-validation-test - description: CRDs have validation -- name: "olm-crds-have-resources" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - olm-crds-have-resources labels: suite: olm test: olm-crds-have-resources-test - description: CRDs have resources -- name: "olm-spec-descriptors" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - olm-spec-descriptors labels: suite: olm test: olm-spec-descriptors-test - description: OLM Spec Descriptors -- name: "olm-status-descriptors" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - olm-status-descriptors labels: suite: olm test: olm-status-descriptors-test - description: OLM Status Descriptors diff --git a/internal/scorecard/alpha/testpod.go b/internal/scorecard/alpha/testpod.go index 89e93d2c17..c1abc96315 100644 --- a/internal/scorecard/alpha/testpod.go +++ b/internal/scorecard/alpha/testpod.go @@ -32,6 +32,7 @@ const ( // PodLabelsDir is the name of the directory containing bundle labels. PodLabelsDirName = "labels" + // PodLabelsDir is the directory containing an annotations.yaml file that is // the source of truth for bundle metadata. These labels come from the // bundle image if applicable. @@ -127,19 +128,18 @@ func getPodDefinition(configMapName string, test Test, r PodTestRunner) *v1.Pod } // getPodLog fetches the test results which are found in the pod log -func getPodLog(ctx context.Context, client kubernetes.Interface, pod *v1.Pod) (logOutput []byte, err error) { - +func getPodLog(ctx context.Context, client kubernetes.Interface, pod *v1.Pod) ([]byte, error) { req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &v1.PodLogOptions{}) podLogs, err := req.Stream(ctx) if err != nil { - return logOutput, err + return nil, err } defer podLogs.Close() buf := new(bytes.Buffer) _, err = io.Copy(buf, podLogs) if err != nil { - return logOutput, err + return nil, err } return buf.Bytes(), err } @@ -154,5 +154,4 @@ func (r PodTestRunner) deletePods(ctx context.Context, configMapName string) err return fmt.Errorf("error deleting pods selector %s %w", selector, err) } return nil - } diff --git a/pkg/apis/scorecard/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/scorecard/v1alpha2/zz_generated.deepcopy.go index 7e864c1730..a181df4ee3 100644 --- a/pkg/apis/scorecard/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/scorecard/v1alpha2/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2019 The Operator-SDK Authors +// Copyright 2020 The Operator-SDK Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Code generated by deepcopy-gen. DO NOT EDIT. +// Code generated by controller-gen. DO NOT EDIT. package v1alpha2 @@ -34,7 +34,6 @@ func (in *ScorecardOutput) DeepCopyInto(out *ScorecardOutput) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScorecardOutput. @@ -58,23 +57,23 @@ func (in *ScorecardOutput) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScorecardTestResult) DeepCopyInto(out *ScorecardTestResult) { *out = *in - if in.Suggestions != nil { - in, out := &in.Suggestions, &out.Suggestions - *out = make([]string, len(*in)) - copy(*out, *in) + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } } if in.Errors != nil { in, out := &in.Errors, &out.Errors *out = make([]string, len(*in)) copy(*out, *in) } - if in.Labels != nil { - out.Labels = make(map[string]string) - for key, value := range in.Labels { - out.Labels[key] = value - } + if in.Suggestions != nil { + in, out := &in.Suggestions, &out.Suggestions + *out = make([]string, len(*in)) + copy(*out, *in) } - return } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScorecardTestResult. diff --git a/pkg/apis/scorecard/v1alpha3/formatter.go b/pkg/apis/scorecard/v1alpha3/formatter.go index 074ad06a84..db6d430787 100644 --- a/pkg/apis/scorecard/v1alpha3/formatter.go +++ b/pkg/apis/scorecard/v1alpha3/formatter.go @@ -30,12 +30,12 @@ const ( noColor = "%s\n" ) -func (s Test) MarshalText() (string, error) { +func (s Test) MarshalText() string { var sb strings.Builder - failColor := ": \033[1;" + redColor + "m%s\033[0m\n" - passColor := ": \033[1;" + greenColor + "m%s\033[0m\n" - warnColor := ": \033[1;" + yellowColor + "m%s\033[0m\n" + failColor := "\033[1;" + redColor + "m%s\033[0m" + passColor := "\033[1;" + greenColor + "m%s\033[0m" + warnColor := "\033[1;" + yellowColor + "m%s\033[0m" // turn off colorization if not in a terminal if !isatty.IsTerminal(os.Stdout.Fd()) && @@ -45,47 +45,60 @@ func (s Test) MarshalText() (string, error) { warnColor = noColor } + sb.WriteString(fmt.Sprintf("%s\n", strings.Repeat("-", 80))) + sb.WriteString(fmt.Sprintf("Image: %s\n", s.Spec.Image)) + + if len(s.Spec.Entrypoint) > 0 { + sb.WriteString(fmt.Sprintf("Entrypoint: %s\n", s.Spec.Entrypoint)) + } + if len(s.Spec.Labels) > 0 { - sb.WriteString("\tLabels: \n") + sb.WriteString("Labels:\n") for labelKey, labelValue := range s.Spec.Labels { - sb.WriteString(fmt.Sprintf("\t\t%q:%q\n", labelKey, labelValue)) + sb.WriteString(fmt.Sprintf("\t%q:%q\n", labelKey, labelValue)) } } - for _, result := range s.Status.Results { - sb.WriteString(fmt.Sprintf("\t%-35s ", result.Name)) - if result.State == PassState { - sb.WriteString(fmt.Sprintf(passColor, PassState)) - } else if result.State == FailState { - sb.WriteString(fmt.Sprintf(failColor, FailState)) - } else if result.State == ErrorState { - sb.WriteString(fmt.Sprintf(failColor, ErrorState)) - } else { + if len(s.Status.Results) > 0 { + sb.WriteString("Results:\n") + for _, result := range s.Status.Results { + if len(result.Name) > 0 { + sb.WriteString(fmt.Sprintf("\tName: %s\n", result.Name)) + } + sb.WriteString("\tState: ") + if result.State == PassState { + sb.WriteString(fmt.Sprintf(passColor, PassState)) + } else if result.State == FailState { + sb.WriteString(fmt.Sprintf(failColor, FailState)) + } else if result.State == ErrorState { + sb.WriteString(fmt.Sprintf(failColor, ErrorState)) + } else { + sb.WriteString("unknown") + } sb.WriteString("\n") - } - if len(result.Suggestions) > 0 { - sb.WriteString(fmt.Sprintf(warnColor, "Suggestions:")) - } - for _, suggestion := range result.Suggestions { - sb.WriteString(fmt.Sprintf("\t\t%s\n", suggestion)) - } + if len(result.Suggestions) > 0 { + sb.WriteString(fmt.Sprintf(warnColor, "\tSuggestions:\n")) + for _, suggestion := range result.Suggestions { + sb.WriteString(fmt.Sprintf("\t\t%s\n", suggestion)) + } + } - if len(result.Errors) > 0 { - sb.WriteString(fmt.Sprintf(failColor, "Errors:")) + if len(result.Errors) > 0 { + sb.WriteString(fmt.Sprintf(failColor, "\tErrors:\n")) + for _, err := range result.Errors { + sb.WriteString(fmt.Sprintf("\t\t%s\n", err)) + } + } - } - for _, err := range result.Errors { - sb.WriteString(fmt.Sprintf("\t\t%s\n", err)) - } - if result.Log != "" { - sb.WriteString("\tLog:\n") - scanner := bufio.NewScanner(strings.NewReader(result.Log)) - for scanner.Scan() { - sb.WriteString(fmt.Sprintf("\t\t%s\n", scanner.Text())) + if result.Log != "" { + sb.WriteString("\tLog:\n") + scanner := bufio.NewScanner(strings.NewReader(result.Log)) + for scanner.Scan() { + sb.WriteString(fmt.Sprintf("\t\t%s\n", scanner.Text())) + } } + sb.WriteString("\n") } - sb.WriteString("\n") } - - return sb.String(), nil + return sb.String() } diff --git a/pkg/apis/scorecard/v1alpha3/types.go b/pkg/apis/scorecard/v1alpha3/types.go index 6c5b6e19af..2096792e86 100644 --- a/pkg/apis/scorecard/v1alpha3/types.go +++ b/pkg/apis/scorecard/v1alpha3/types.go @@ -34,8 +34,8 @@ const ( type TestSpec struct { // Image is the name of the testimage Image string `json:"image"` - // EntryPoint is list of commands and arguments passed to the test image - EntryPoint []string `json:"entrypoint,omitempty"` + // Entrypoint is list of commands and arguments passed to the test image + Entrypoint []string `json:"entrypoint,omitempty"` // Labels that further describe the test and enable selection Labels map[string]string `json:"labels,omitempty"` } @@ -43,7 +43,7 @@ type TestSpec struct { // TestResult contains the results of an individual scorecard test type TestResult struct { // Name is the name of the test - Name string `json:"name"` + Name string `json:"name,omitempty"` // Log holds a log produced from the test (if applicable) Log string `json:"log,omitempty"` // State is the final state of the test @@ -56,23 +56,38 @@ type TestResult struct { // TestStatus contains collection of testResults. type TestStatus struct { - Results []TestResult `json:"results"` + Results []TestResult `json:"results,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Test is the schema for the scorecard API +// Test specifies a single test run. type Test struct { metav1.TypeMeta `json:",inline"` Spec TestSpec `json:"spec,omitempty"` Status TestStatus `json:"status,omitempty"` } -func NewTest() *Test { - return &Test{ +// TestList is a list of tests. +type TestList struct { + metav1.TypeMeta `json:",inline"` + Items []Test `json:"items"` +} + +func NewTest() Test { + return Test{ TypeMeta: metav1.TypeMeta{ + APIVersion: SchemeGroupVersion.String(), Kind: "Test", + }, + } +} + +func NewTestList() TestList { + return TestList{ + TypeMeta: metav1.TypeMeta{ APIVersion: SchemeGroupVersion.String(), + Kind: "TestList", }, } } diff --git a/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go index bcddf16179..0ba9be02e9 100644 --- a/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go +++ b/pkg/apis/scorecard/v1alpha3/zz_generated.deepcopy.go @@ -1,20 +1,18 @@ // +build !ignore_autogenerated -/* - - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ +// Copyright 2020 The Operator-SDK Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Code generated by controller-gen. DO NOT EDIT. @@ -50,6 +48,29 @@ func (in *Test) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestList) DeepCopyInto(out *TestList) { + *out = *in + out.TypeMeta = in.TypeMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Test, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestList. +func (in *TestList) DeepCopy() *TestList { + if in == nil { + return nil + } + out := new(TestList) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestResult) DeepCopyInto(out *TestResult) { *out = *in @@ -78,8 +99,8 @@ func (in *TestResult) DeepCopy() *TestResult { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestSpec) DeepCopyInto(out *TestSpec) { *out = *in - if in.EntryPoint != nil { - in, out := &in.EntryPoint, &out.EntryPoint + if in.Entrypoint != nil { + in, out := &in.Entrypoint, &out.Entrypoint *out = make([]string, len(*in)) copy(*out, *in) } diff --git a/website/content/en/docs/scorecard/custom-tests.md b/website/content/en/docs/scorecard/custom-tests.md index 40b54253e7..4552341a65 100644 --- a/website/content/en/docs/scorecard/custom-tests.md +++ b/website/content/en/docs/scorecard/custom-tests.md @@ -87,16 +87,14 @@ The [configuration file][config_yaml] includes test definitions and metadata to ```yaml tests: -- name: "customtest1" - image: quay.io/username/custom-scorecard-tests:dev +- image: quay.io/username/custom-scorecard-tests:dev entrypoint: - custom-scorecard-tests - customtest1 labels: suite: custom test: customtest1 - description: an ISV custom test - ``` +``` The important fields to note here are: 1. `image` - name and tag of the test image which was specified in the Makefile. diff --git a/website/content/en/docs/scorecard/kuttl-tests.md b/website/content/en/docs/scorecard/kuttl-tests.md index 207473b3ec..91f552b863 100644 --- a/website/content/en/docs/scorecard/kuttl-tests.md +++ b/website/content/en/docs/scorecard/kuttl-tests.md @@ -49,12 +49,10 @@ In the scorecard configuration file, you might have the following definition of what the selector `suite=kuttlsuite` will translate to: ```yaml tests: -- name: "kuttltest1" - image: quay.io/operator-framework/scorecard-test-kuttl:dev +- image: quay.io/operator-framework/scorecard-test-kuttl:dev labels: suite: kuttlsuite test: kuttltest1 - description: an ISV custom test that does... ``` This test configuration will execute the scorecard-test-kuttl diff --git a/website/content/en/docs/scorecard/scorecard-alpha.md b/website/content/en/docs/scorecard/scorecard-alpha.md index 365f7db8c5..550c817d4f 100644 --- a/website/content/en/docs/scorecard/scorecard-alpha.md +++ b/website/content/en/docs/scorecard/scorecard-alpha.md @@ -61,24 +61,20 @@ A sample of the scorecard configuration file may look as follows: ```yaml tests: -- name: "basic-check-spec" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - basic-check-spec labels: suite: basic test: basic-check-spec-test - description: check the spec test -- name: "olm-bundle-validation" - image: quay.io/operator-framework/scorecard-test:dev +- image: quay.io/operator-framework/scorecard-test:dev entrypoint: - scorecard-test - olm-bundle-validation labels: suite: olm test: olm-bundle-validation-test - description: validate the bundle test ``` The configuration file defines each test that scorecard can execute. The @@ -160,20 +156,34 @@ See an example of the JSON format produced by a scorecard test: ```json { - "spec": { - "image": "" - }, - "status": { - "results": [ - { - "name": "olm-bundle-validation", - "log": "time=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found metadata directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level -=debug msg=\"Getting mediaType info from manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Found annotations file\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Could not find optio -nal dependencies file\" name=bundle-test\n", - "state": "pass" + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "kind": "TestList", + "items": [ + { + "kind": "Test", + "apiVersion": "scorecard.operatorframework.io/v1alpha3", + "spec": { + "image": "quay.io/operator-framework/scorecard-test:dev", + "entrypoint": [ + "scorecard-test", + "olm-bundle-validation" + ], + "labels": { + "suite": "olm", + "test": "olm-bundle-validation-test" + } + }, + "status": { + "results": [ + { + "name": "olm-bundle-validation", + "log": "time=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=debug msg=\"Found metadata directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=debug msg=\"Getting mediaType info from manifests directory\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Found annotations file\" name=bundle-test\ntime=\"2020-06-10T19:02:49Z\" level=info msg=\"Could not find optional dependencies file\" name=bundle-test\n", + "state": "pass" + } + ] } - ] - } + } + ] } ``` @@ -182,17 +192,24 @@ nal dependencies file\" name=bundle-test\n", See an example of the text format produced by a scorecard test: ``` - Labels: - olm-bundle-validation : pass - Log: - time="2020-06-10T19:00:43Z" level=debug msg="Found manifests directory" name=bundle-test - time="2020-06-10T19:00:43Z" level=debug msg="Found metadata directory" name=bundle-test - time="2020-06-10T19:00:43Z" level=debug msg="Getting mediaType info from manifests directory" name=bundle-test - time="2020-06-10T19:00:43Z" level=info msg="Found annotations file" name=bundle-test - time="2020-06-10T19:00:43Z" level=info msg="Could not find optional dependencies file" name=bundle-test +-------------------------------------------------------------------------------- +Image: quay.io/operator-framework/scorecard-test:dev +Entrypoint: [scorecard-test olm-bundle-validation] +Labels: + "suite":"olm" + "test":"olm-bundle-validation-test" +Results: + Name: olm-bundle-validation + State: pass + Log: + time="2020-07-15T03:19:02Z" level=debug msg="Found manifests directory" name=bundle-test + time="2020-07-15T03:19:02Z" level=debug msg="Found metadata directory" name=bundle-test + time="2020-07-15T03:19:02Z" level=debug msg="Getting mediaType info from manifests directory" name=bundle-test + time="2020-07-15T03:19:02Z" level=info msg="Found annotations file" name=bundle-test + time="2020-07-15T03:19:02Z" level=info msg="Could not find optional dependencies file" name=bundle-test ``` -**NOTE** The output format spec matches the [`Test`](https://godoc.org/github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3#Test) type layout. +**NOTE** The output format spec for each test matches the [`Test`](https://godoc.org/github.com/operator-framework/operator-sdk/pkg/apis/scorecard/v1alpha3#Test) type layout. ## Exit Status From 1df24edc4332dd13e1cdeaaed0f70a6f2d59eab1 Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Wed, 15 Jul 2020 08:26:18 -0400 Subject: [PATCH 2/2] internal/scorecard/alpha: fix bundle.tar.gz testdata --- .../scorecard/alpha/testdata/bundle.tar.gz | Bin 2840 -> 2784 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/internal/scorecard/alpha/testdata/bundle.tar.gz b/internal/scorecard/alpha/testdata/bundle.tar.gz index f79410dadcf527f29bac3900d3a00a8ecc8fc32c..5e0933ba70ce42e96b559c4ff79b9d16b68f66b5 100644 GIT binary patch literal 2784 zcmV<63Lo_!iwFP!00000|LhxGbK|zr&-xWR?t^FAkP`Ko@=f(Qsi${7jpe2fnamAD zt|Y`DzyP4`bm{%~4#00pvg9vaF4aC*))H7O_Ip7nBAm`qX?6B`(CKtKqrt#@dZR(7 z`kBL-*Bf)(-5?%a6 zqL`r@i0DIvkwu2;Lm)$UO(OQBHH*5huNR!hYgn4t?PlGumM0Vx$_#sH*izsm_)9HUDAtB!w5mA21fmPo>?ELgOn ze339BD-T)#DiAUHaATH>Ns!`_nauW>v7j{3iSpt1FD-x-VKgLW-|0gUBOgz$f9g$x zB}ODg0)(gnNn<;%OWA=h?O6|!K45DsGYD+bzyDM~{G)KVlYgh22WN(~YN zvEcM`L0N$oc1cDwW?IRCa*dJ`X2h7_cR+Xu5m|#ovj!x*Ld_0QF1!;Gf%3WVVX1Yj z{C0amb#9}9h@ym3y>17BYe{E`7E-lCTp?>Kx^RdLmQ-V)6NzmSQ^zvSwKeo8{7a$i ziwdWDZR|p6N%^8cTT|RAcVm!7za*QM@?VPApo~;*ucmj9FKSg#m${X8IZc&|V1a&5l^j3TcKT_v-KYNP4j5)0D$sdo*{Y$!3#Ty#HQxwPQEBW~6OG2|>)~d%Kc@fOu^Y)K46K`C>ulpuA^8;O}O^q zHsNq?7$RCoM}_yWKyJPv+<&jmjSEUFP=N>$$b1{kNQ890PL0zT6=ewHFoeKH`%1Wj z#KcgzPi8nvC<~1ll7(t+9SUoi332bNJlA}>m1)L&X1EqS{4gVm2CAL$N%_JS8%rIr z4XRW68T0UIM4ra4O6-IXVm?`7yPqaiX!Y1kfQ@k7B{d<}kqqRlDO@omQWt2AhWSiWeS=#7x zY;uFFwb56y$r~eZAQG+eY|*l5ma(WuI0IB;w8e9_2EXbfA|+YxHBxH_4_j#iwaUxD zl}A{-BwmTLY&uz8S9uJOT8Ix?CYaSpLXrh$n6F*ko?su8{a<0NPVT;L=HEuiG{DRl z9^VH(-2ZuTf1LAo?$;Ir zI5#oPOw5*W3NJXXjQdg}W(v$rLZ&J>_o26YnoadvI53g=+y|bpYIgZj{##(bWdHqQ zKLZkz8D-QK0q_r#vf!F4O)Q;86J9#?s^fNCFK@6-$LmAPNPtUWlV@H(lbB|X9~NzX z-T

W znU;-8MZ~*&c4u!ZF=jL%$_KB4RmhAmWWBhIhz^!-$qd;>)wy?FV;H7jU#znLkb}l3 zW<)od-)+6R0k(DZgVx{NJJ^1KYeG574WbSNB8o`;uqJ=BXC(v3SKDT$oc?Egclpl` zH{*9#)5-YqYGWLv=RQA35p5DNFsI1ETbyrynmwObx5#DbR`Tl47lK8uCUSwgIJZ<6 zzwBgq^8Srw_c)xIHFvV|esXm?zI%WBAU}JJ!2peHqj7sRc{{$m`f&Yj{NpA;J7|(H zHW7>lYahn!BU!7bnIO~xYuytd>xO86;~+4h_~v1fG0KQ4Rc1{iBbhL)g*y_{AEiiQ zRd4Uk&s*DAs*S0IUwd$ovbP=Co>NX3{fx5yJU>{hV`Gy!lt!*sf+Z0KjL@htafQuU zf#7qxh)CRIA&^)bx?V*{KQS;FiD3h^KwbOJsH!7k69sFS^yssO`dAyZQwb!l zZDfZUDv`d9Cm0E?uz}7)dJ}ICVde=_-=d@j2iwZE>_Pa7hA8t_vw2ICcDY~sc;ycddC~To zYdq%uPv79Zt~ctR?*AMEa{ad}?Zwk?`fP_i_kTLWUaz75L4R~Q|2+ycPHW2};QYV2 zBoTfT^4_Jwoh3YE=oY6O?#lB);({tI*MGC8NC#5LUprU|!4Pj>WEhoBJe&H)f`kzjnAf!3#q( z|KBHCzhv)!_Ne;5>-7ev{_78i!&Cit4A`pw^sn{!m#75yo&R{n{l8rQqtT#ys{f7w z)%$<9%-J#M`lPFp-k%Td1KJ;6fMko^PgV*h}j12Xq;93ecVT5D;e&B0w z?*w~byZ}v~;CbISR#0y*` md5POB&%TcGLKjlLwwavZ1Sj|t!v6vQ0RR6iR6QpEPyhfxopyHs literal 2840 zcmV+z3+MD7iwFP!00000|Li~8c;2rZ9-Mb*Jb#e<^BV!H{|;u1E361- z(%s;gzUUrW9>acrul|Rl28ClTUbT>llt2Q7qn0s<>8q?`Y5E^rJVxmdY` zyR(!<6xDJE`!D!ec485zKK)l_8XoIEnrX(ZAvzN5}u3 zKj>>0^1J%u9RE)MrRy0VaN!e<6K$6~{!loE{1A%h>3QKm{2%zeVWIz?wv(&=z2D^o4UQ72{#D1nB~qEOgr)-1 zDhmcJ30tHTi^@O?AsO-n0(7glOK@m$=}ctLn6V%gR;dio?_XL7tuZB$Nt$+$C&0#& zo1c2qa0xNCNQ5Ggp&*HwS8|DnL~04h&P8)B%-51vG@jgK6cV9xhax!_8Y&tqaE-k# zH8XY&inNZvZn?HNh>*0K^EpyWB9VYZfCOeMfeKlqr(lNo>{keti>4qn5hRy9r4b4_ zTZ2$YK*$%2d@d+Uq`29VVg;F2av)5BV3=w#rtlrYEJ88fAOU(0l(GsnGex@S9T$KI zoAUrIl}cpL=`4uKZ8YR@oHC*|oscsn$Sl=)w-dn{Xh)KTgGIO`3PP0%=-`AnhH+-= z-8lNcLfMxUPIZB!ld&LdQJ{?}?v=YXNUdL-EwlWW;?*cEmAmWdJ<2yVDyYlcNV{An zOTlYF5jF>56*m_=HWYzH2`5aMFCits>Oz<4EG9~8`nObvd!)FFu1xiZW`NQp!U`f6 z-7s{8W1v@9!mEp2OISKuq=yAx4KVeIR$2u*4=p=2RCwVufw1Nq5lAAm{VJ?LTYWv; z?Dt0-Sw2G@)3l76+v)Y)JsQ7xb8~<5{&xHpO()k^=!cu@w{ND%+1@^+sZ<~uL=n1) z6D|~HDnNHBL+!k{bY!$b3t;*ipw;)XtzA$+feLv92+w!XjCn-n8*7}}s0al#jv^GA zXkQ6;DAh3(tHkB$m!nQ^{y*1oGh-O4g;J~P;G7JZmuNkZAl_#|v$ zijAd;=@!)~`3!mZ)FQX>s}eg=1PSGv7?`?jte$fbV-@J=)9W_ItR3mNw5Ul(JB!vz z_4ypL^FbrD>Ef^%nt3%t3QbuW`PqxV| znx&0C$0oPP8XJ8jo4nNmM_7PSo+(GYz^-G*#)5+?($}mFKLIR{j3R#`R1ztdg`P$X(84f|&|7{` zV|aWY_;CN{d%fPM(f{=Z!{NFAI|Wqxzw*Q?5Nhj~ zrWa;gIE5FqEAzh8i0J}zn~<&w+5zhAUuIJM4i0prZU=~^w3=PMl>aTTU$X!GVm?Eh z;2ELB6anZTI%T07CKWbx8clfV)N9Z6Tt9EHb;lc^gyImE+$7Jue#QyO96u`B{Ja6S zNjPs{s^hjF{>butXgAw+?Ci+)+oHmnB$^I#SM<{E4=|r;~5B@-pkZZ zw8U8mrsK8cr&}m2A~KN%{gOJCr?yNW? zp_`H*;n5XO%d)p|e4)u$F-~|{Mph(~%fuMzVW}Kl_3ReKZLK=7vZz-q&k}|qb%|;o zRAYZq+Iw}N)nK!<*7xO`0TSAi7YEI-gkj&rJ=C`GqIffsVTkp|>

WnU;-8Ma27j zc5iPhAfY70GC+O>D}icZhYN}PbcH6 z>#cdnp8NbDc)U$S!<+z(?qI%qG#j57w6~!eRlcIJcCSzwBjr^8Srs z_cWa9J@>NmesX;`zJGuBAU_9A(_#kUWWOhCj$$}($e8HtoaE!^RR{3v*u$a;I%Znt)^ zRGU)^zvke=vbP)AnG=R7`3#~yo*yjMv5C$cQUOzIPSY4dib>p9xQ6Dckh3{i#5ifP z5DKVGU4xi4o+cDfKQYi5i9!Rl09E@=iL4_M9R+I`d-PdDeXLE|s|1|X9%PRiDv`d9 z#}IQSp@GgrdJ}IBVdg27-=d^q&WPfIutlY{I4ru%HnY_S9=rKK4KX}ci?`+Lr{XCd ziVlahzuUuhJV`^&FBt?5$nBY?zRz}dyTDU<*;nU3z}V&gU;a^j<`FrgtieiwmNn+P z{N&d`GEEXLRPA8S7Nze$vZ7!(Fm+1Ug2LX}b4+D*`n!}WaxCpJrDfgHLTCk>k6c6< z`^K(gOsch=W3z3HRVs+uWVdZv<{*4cA`tnj*}SE(UGCRDUirgAUbMaD9*?^JGaOxb zez)uO&-Z^$0maq1SG)mw^!-n*e~tGC{n7dS_atz>|M}Ox|EXU2?&rnP@qaiR_WRwT z*Sqk1e=r=Ke0es9n_=l_#HL;oFnMDwy0z=8bV z^#-Gc{s(&aoc~V%M*l&}$xl;%OYz1fyi;7^DUw!cB#vGbM6hll1WbvIIUo#>ZTf1n z^7kYpbiyTsD@P^}w(4&JWJ(kSc9!iCHTkVomfou}%4zJ_yzG?si*|F%X) z9EwOfOT2yx&%4&1obBSw=J8w?OHahKvBS!X$zwTgt~(XG=56ki7~WcGDSz#7b%qy) zX8vELO1$S8D)fHz-a5|xw zLH>T=Yd_u@4!~~zr$j0z;nB;S3_U*n4~M>gj{m2C1N!fHo`aj0|LBhz`tN(*spJ21>3pO5-}<%Qv#%{Yn#j)E q6EAR)%NJ0RR64*y#EIRsaCAk(H_d