diff --git a/conformance/01_pull_test.go b/conformance/01_pull_test.go index 23100cdb..2c9b49bf 100644 --- a/conformance/01_pull_test.go +++ b/conformance/01_pull_test.go @@ -152,6 +152,19 @@ var test01Pull = func() { }) g.Context("Teardown", func() { + if deleteManifestBeforeBlobs { + g.Specify("Delete manifest created in setup", func() { + SkipIfDisabled(pull) + RunOnlyIf(runPullSetup) + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(SatisfyAll( + BeNumerically(">=", 200), + BeNumerically("<", 300))) + }) + } + g.Specify("Delete config blob created in setup", func() { SkipIfDisabled(pull) RunOnlyIf(runPullSetup) @@ -174,16 +187,18 @@ var test01Pull = func() { BeNumerically("<", 300))) }) - g.Specify("Delete manifest created in setup", func() { - SkipIfDisabled(pull) - RunOnlyIf(runPullSetup) - req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) - resp, err := client.Do(req) - Expect(err).To(BeNil()) - Expect(resp.StatusCode()).To(SatisfyAll( - BeNumerically(">=", 200), - BeNumerically("<", 300))) - }) + if !deleteManifestBeforeBlobs { + g.Specify("Delete manifest created in setup", func() { + SkipIfDisabled(pull) + RunOnlyIf(runPullSetup) + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(SatisfyAll( + BeNumerically(">=", 200), + BeNumerically("<", 300))) + }) + } }) }) } diff --git a/conformance/02_push_test.go b/conformance/02_push_test.go index d4c6a43e..7d790977 100644 --- a/conformance/02_push_test.go +++ b/conformance/02_push_test.go @@ -246,6 +246,23 @@ var test02Push = func() { }) g.Context("Teardown", func() { + if deleteManifestBeforeBlobs { + g.Specify("Delete manifest created in tests", func() { + SkipIfDisabled(push) + RunOnlyIf(runPushSetup) + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(SatisfyAny( + SatisfyAll( + BeNumerically(">=", 200), + BeNumerically("<", 300), + ), + Equal(http.StatusMethodNotAllowed), + )) + }) + } + g.Specify("Delete config blob created in tests", func() { SkipIfDisabled(push) RunOnlyIf(runPushSetup) @@ -276,20 +293,23 @@ var test02Push = func() { )) }) - g.Specify("Delete manifest created in tests", func() { - SkipIfDisabled(push) - RunOnlyIf(runPushSetup) - req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) - resp, err := client.Do(req) - Expect(err).To(BeNil()) - Expect(resp.StatusCode()).To(SatisfyAny( - SatisfyAll( - BeNumerically(">=", 200), - BeNumerically("<", 300), - ), - Equal(http.StatusMethodNotAllowed), - )) - }) + if !deleteManifestBeforeBlobs { + g.Specify("Delete manifest created in tests", func() { + SkipIfDisabled(push) + RunOnlyIf(runPushSetup) + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(SatisfyAny( + SatisfyAll( + BeNumerically(">=", 200), + BeNumerically("<", 300), + ), + Equal(http.StatusMethodNotAllowed), + )) + }) + } + }) }) } diff --git a/conformance/03_discovery_test.go b/conformance/03_discovery_test.go index 2eae1190..0bad4677 100644 --- a/conformance/03_discovery_test.go +++ b/conformance/03_discovery_test.go @@ -128,6 +128,19 @@ var test03ContentDiscovery = func() { }) g.Context("Teardown", func() { + if deleteManifestBeforeBlobs { + g.Specify("Delete created manifest & associated tags", func() { + SkipIfDisabled(contentDiscovery) + RunOnlyIf(runContentDiscoverySetup) + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(SatisfyAll( + BeNumerically(">=", 200), + BeNumerically("<", 300))) + }) + } + g.Specify("Delete config blob created in tests", func() { SkipIfDisabled(contentDiscovery) RunOnlyIf(runContentDiscoverySetup) @@ -150,16 +163,18 @@ var test03ContentDiscovery = func() { BeNumerically("<", 300))) }) - g.Specify("Delete created manifest & associated tags", func() { - SkipIfDisabled(contentDiscovery) - RunOnlyIf(runContentDiscoverySetup) - req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) - resp, err := client.Do(req) - Expect(err).To(BeNil()) - Expect(resp.StatusCode()).To(SatisfyAll( - BeNumerically(">=", 200), - BeNumerically("<", 300))) - }) + if !deleteManifestBeforeBlobs { + g.Specify("Delete created manifest & associated tags", func() { + SkipIfDisabled(contentDiscovery) + RunOnlyIf(runContentDiscoverySetup) + req := client.NewRequest(reggie.DELETE, "/v2//manifests/", reggie.WithDigest(manifestDigest)) + resp, err := client.Do(req) + Expect(err).To(BeNil()) + Expect(resp.StatusCode()).To(SatisfyAll( + BeNumerically(">=", 200), + BeNumerically("<", 300))) + }) + } }) }) } diff --git a/conformance/README.md b/conformance/README.md index 5207dc70..1ddca710 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -30,6 +30,7 @@ export OCI_TEST_CONTENT_MANAGEMENT=1 # Extra settings export OCI_HIDE_SKIPPED_WORKFLOWS=0 export OCI_DEBUG=0 +export OCI_DELETE_MANIFEST_BEFORE_BLOBS=0 ``` Lastly, run the tests: @@ -147,6 +148,15 @@ the report, you must set the following in the environment: OCI_HIDE_SKIPPED_WORKFLOWS=1 ``` +#### Teardown Order + +By default, the teardown phase of each test deletes blobs before manifests. Some registries require the opposite order, deleting manifests before blobs. In this case, you must set the following in the environment: + +``` +# Required to delete manifests before blobs +OCI_DELETE_MANIFEST_BEFORE_BLOBS=1 +``` + #### Container Image You may use the [Dockerfile](./Dockerfile) located in this directory @@ -172,6 +182,7 @@ docker run --rm \ -e OCI_TEST_CONTENT_MANAGEMENT=1 \ -e OCI_HIDE_SKIPPED_WORKFLOWS=0 \ -e OCI_DEBUG=0 \ + -e OCI_DELETE_MANIFEST_BEFORE_BLOBS=0 \ conformance:latest ``` @@ -206,6 +217,7 @@ jobs: OCI_TEST_CONTENT_MANAGEMENT: 1 OCI_HIDE_SKIPPED_WORKFLOWS: 0 OCI_DEBUG: 0 + OCI_DELETE_MANIFEST_BEFORE_BLOBS: 0 - run: mkdir -p .out/ && mv {report.html,junit.xml} .out/ if: always() - name: Upload test results zip as build artifact diff --git a/conformance/setup.go b/conformance/setup.go index 8dadc051..99f4eb01 100644 --- a/conformance/setup.go +++ b/conformance/setup.go @@ -39,23 +39,24 @@ const ( DENIED UNSUPPORTED - envVarRootURL = "OCI_ROOT_URL" - envVarNamespace = "OCI_NAMESPACE" - envVarUsername = "OCI_USERNAME" - envVarPassword = "OCI_PASSWORD" - envVarDebug = "OCI_DEBUG" - envVarPull = "OCI_TEST_PULL" - envVarPush = "OCI_TEST_PUSH" - envVarContentDiscovery = "OCI_TEST_CONTENT_DISCOVERY" - envVarContentManagement = "OCI_TEST_CONTENT_MANAGEMENT" - envVarPushEmptyLayer = "OCI_SKIP_EMPTY_LAYER_PUSH_TEST" - envVarBlobDigest = "OCI_BLOB_DIGEST" - envVarManifestDigest = "OCI_MANIFEST_DIGEST" - envVarEmptyLayerManifestDigest = "OCI_EMPTY_LAYER_MANIFEST_DIGEST" - envVarTagName = "OCI_TAG_NAME" - envVarTagList = "OCI_TAG_LIST" - envVarHideSkippedWorkflows = "OCI_HIDE_SKIPPED_WORKFLOWS" - envVarAuthScope = "OCI_AUTH_SCOPE" + envVarRootURL = "OCI_ROOT_URL" + envVarNamespace = "OCI_NAMESPACE" + envVarUsername = "OCI_USERNAME" + envVarPassword = "OCI_PASSWORD" + envVarDebug = "OCI_DEBUG" + envVarPull = "OCI_TEST_PULL" + envVarPush = "OCI_TEST_PUSH" + envVarContentDiscovery = "OCI_TEST_CONTENT_DISCOVERY" + envVarContentManagement = "OCI_TEST_CONTENT_MANAGEMENT" + envVarPushEmptyLayer = "OCI_SKIP_EMPTY_LAYER_PUSH_TEST" + envVarBlobDigest = "OCI_BLOB_DIGEST" + envVarManifestDigest = "OCI_MANIFEST_DIGEST" + envVarEmptyLayerManifestDigest = "OCI_EMPTY_LAYER_MANIFEST_DIGEST" + envVarTagName = "OCI_TAG_NAME" + envVarTagList = "OCI_TAG_LIST" + envVarHideSkippedWorkflows = "OCI_HIDE_SKIPPED_WORKFLOWS" + envVarAuthScope = "OCI_AUTH_SCOPE" + envVarDeleteManifestBeforeBlobs = "OCI_DELETE_MANIFEST_BEFORE_BLOBS" emptyLayerTestTag = "emptylayer" testTagName = "tagtest0" @@ -123,6 +124,7 @@ var ( runContentDiscoverySetup bool runContentManagementSetup bool skipEmptyLayerTest bool + deleteManifestBeforeBlobs bool Version = "unknown" ) @@ -274,6 +276,7 @@ func init() { runContentDiscoverySetup = true runContentManagementSetup = true skipEmptyLayerTest = false + deleteManifestBeforeBlobs = false if os.Getenv(envVarTagName) != "" && os.Getenv(envVarManifestDigest) != "" && @@ -286,6 +289,7 @@ func init() { } skipEmptyLayerTest, _ = strconv.ParseBool(os.Getenv(envVarPushEmptyLayer)) + deleteManifestBeforeBlobs, _ = strconv.ParseBool(os.Getenv(envVarDeleteManifestBeforeBlobs)) reportJUnitFilename = "junit.xml" reportHTMLFilename = "report.html"