fix(test): E2E test cases for OCI layout (notaryproject#692)

test/e2e/internal/notation/host.go

@@ -15,12 +15,12 @@ import (
 // vhost is the VirtualHost instance.
 type CoreTestFunc func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost)
 
-// GeneralTestFunc is the test function running in a VirtualHost agnostic to
-// the Repository of the artifact.
+// OCILayoutTestFunc is the test function running in a VirtualHost with isolated
+// OCI layout for each test case.
 //
 // notation is an Executor isolated by $XDG_CONFIG_HOME.
 // vhost is the VirtualHost instance.
-type GeneralTestFunc func(notation *utils.ExecOpts, vhost *utils.VirtualHost)
+type OCILayoutTestFunc func(notation *utils.ExecOpts, ocilayout *OCILayout, vhost *utils.VirtualHost)
 
 // Host creates a virtualized notation testing host by modify
 // the "XDG_CONFIG_HOME" environment variable of the Executor.
@@ -52,21 +52,26 @@ func HostInGithubAction(options []utils.HostOption, fn CoreTestFunc) {
 	Host(options, fn)
 }
 
-// GeneralHost creates a virtualized notation testing host by modify
-// the "XDG_CONFIG_HOME" environment variable of the Executor. It's agnostic to
-// the Repository of the artifact.
+// HostWithOCILayout creates a virtualized notation testing host by modify
+// the "XDG_CONFIG_HOME" environment variable of the Executor. It generates
+// isolated OCI layout in the testing host.
 //
 // options is the required testing environment options
 // fn is the callback function containing the testing logic.
-func GeneralHost(options []utils.HostOption, fn GeneralTestFunc) {
+func HostWithOCILayout(options []utils.HostOption, fn OCILayoutTestFunc) {
 	// create a notation vhost
 	vhost, err := createNotationHost(NotationBinPath, options...)
 	if err != nil {
 		panic(err)
 	}
 
+	ocilayout, err := GenerateOCILayout("")
+	if err != nil {
+		panic(err)
+	}
+
 	// run the main logic
-	fn(vhost.Executor, vhost)
+	fn(vhost.Executor, ocilayout, vhost)
 }
 
 // OldNotation create an old version notation ExecOpts in a VirtualHost

test/e2e/internal/notation/init.go

@@ -29,7 +29,6 @@ const (
 	envKeyNotationPluginPath = "NOTATION_E2E_PLUGIN_PATH"
 	envKeyNotationConfigPath = "NOTATION_E2E_CONFIG_PATH"
 	envKeyOCILayoutPath      = "NOTATION_E2E_OCI_LAYOUT_PATH"
-	envKeyOCILayoutTestPath  = "NOTATION_E2E_OCI_LAYOUT_TEST_PATH"
 	envKeyTestRepo           = "NOTATION_E2E_TEST_REPO"
 	envKeyTestTag            = "NOTATION_E2E_TEST_TAG"
 )
@@ -49,7 +48,6 @@ var (
 
 var (
 	OCILayoutPath       string
-	OCILayoutTestPath   string
 	TestRepoUri         string
 	TestTag             string
 	RegistryStoragePath string
@@ -58,7 +56,6 @@ var (
 func init() {
 	RegisterFailHandler(Fail)
 	setUpRegistry()
-	setUpOCILayout()
 	setUpNotationValues()
 }
 
@@ -73,10 +70,6 @@ func setUpRegistry() {
 	setValue(envKeyTestTag, &TestTag)
 }
 
-func setUpOCILayout() {
-	setPathValue(envKeyOCILayoutTestPath, &OCILayoutTestPath)
-}
-
 func setUpNotationValues() {
 	// set Notation binary path
 	setPathValue(envKeyNotationBinPath, &NotationBinPath)

test/e2e/internal/notation/layout.go

@@ -0,0 +1,64 @@
+package notation
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	"oras.land/oras-go/v2"
+	"oras.land/oras-go/v2/content/oci"
+)
+
+// OCILayout is a OCI layout directory for
+type OCILayout struct {
+	// Path is the path of the OCI layout directory.
+	Path string
+	// Tag is the tag of artifact in the OCI layout.
+	Tag string
+	// Digest is the digest of artifact in the OCI layout.
+	Digest string
+}
+
+// GenerateOCILayout creates a new OCI layout in a temporary directory.
+func GenerateOCILayout(srcRepoName string) (*OCILayout, error) {
+	ctx := context.Background()
+
+	if srcRepoName == "" {
+		srcRepoName = TestRepoUri
+	}
+
+	destPath := filepath.Join(GinkgoT().TempDir(), newRepoName())
+	// create a local store from OCI layout directory.
+	srcStore, err := oci.NewFromFS(ctx, os.DirFS(filepath.Join(OCILayoutPath, srcRepoName)))
+	if err != nil {
+		return nil, err
+	}
+
+	// create a dest store for store the generated oci layout.
+	destStore, err := oci.New(destPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// copy data
+	desc, err := oras.ExtendedCopy(ctx, srcStore, TestTag, destStore, "", oras.DefaultExtendedCopyOptions)
+	if err != nil {
+		return nil, err
+	}
+	return &OCILayout{
+		Path:   destPath,
+		Tag:    TestTag,
+		Digest: desc.Digest.String(),
+	}, nil
+}
+
+// ReferenceWithTag returns the reference with tag.
+func (o *OCILayout) ReferenceWithTag() string {
+	return o.Path + ":" + o.Tag
+}
+
+// ReferenceWithDigest returns the reference with digest.
+func (o *OCILayout) ReferenceWithDigest() string {
+	return o.Path + "@" + o.Digest
+}

test/e2e/run.sh

@@ -92,7 +92,6 @@ trap cleanup EXIT
 # set environment variable for E2E testing
 export NOTATION_E2E_CONFIG_PATH=$CWD/testdata/config
 export NOTATION_E2E_OCI_LAYOUT_PATH=$CWD/testdata/registry/oci_layout
-export NOTATION_E2E_OCI_LAYOUT_TEST_PATH=$CWD/testdata/oci-layout/e2e
 export NOTATION_E2E_TEST_REPO=e2e
 export NOTATION_E2E_TEST_TAG=v1
 export NOTATION_E2E_PLUGIN_PATH=$CWD/plugin/bin/$PLUGIN_NAME

test/e2e/suite/command/inspect.go

@@ -48,8 +48,8 @@ var _ = Describe("notation inspect", func() {
 
 			notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()).
 				MatchKeyWords(inspectSuccessfully...).
-				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
-				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+				MatchErrKeyWords(HTTPSRequest).
+				NoMatchErrKeyWords(HTTPRequest)
 		})
 	})
 

test/e2e/suite/command/list.go

@@ -31,8 +31,8 @@ var _ = Describe("notation list", func() {
 					"└── application/vnd.cncf.notary.signature",
 					"└── sha256:",
 				).
-				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
-				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+				MatchErrKeyWords(HTTPSRequest).
+				NoMatchErrKeyWords(HTTPRequest)
 		})
 	})
 
@@ -50,4 +50,26 @@ var _ = Describe("notation list", func() {
 				NoMatchErrKeyWords(HTTPSRequest)
 		})
 	})
+
+	It("all signatures of an oci-layout", func() {
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, _ *OCILayout, vhost *utils.VirtualHost) {
+			ociLayout, err := GenerateOCILayout("e2e-valid-signature")
+			if err != nil {
+				Fail(err.Error())
+			}
+
+			notation.Exec("list", "--oci-layout", ociLayout.ReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:273243a7a64e9312761ca0aa8f43b6ba805e677a561558143b6e92981c487339",
+				)
+		})
+	})
+
+	It("oci-layout with no signature", func() {
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("list", "--oci-layout", ociLayout.ReferenceWithDigest()).
+				MatchKeyWords("has no associated signature")
+		})
+	})
 })

test/e2e/suite/command/sign.go

@@ -108,45 +108,37 @@ var _ = Describe("notation sign", func() {
 	})
 
 	It("by digest with oci layout", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f"
-			ociLayoutReference := OCILayoutTestPath + "@" + digest
-			notation.Exec("sign", "--oci-layout", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(SignSuccessfully)
 		})
 	})
 
 	It("by digest with oci layout and COSE format", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f"
-			ociLayoutReference := OCILayoutTestPath + "@" + digest
-			notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(SignSuccessfully)
 		})
 	})
 
 	It("by tag with oci layout", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			ociLayoutReference := OCILayoutTestPath + ":" + TestTag
-			notation.Exec("sign", "--oci-layout", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(SignSuccessfully)
 		})
 	})
 
 	It("by tag with oci layout and COSE format", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			ociLayoutReference := OCILayoutTestPath + ":" + TestTag
-			notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(SignSuccessfully)
 		})
 	})
 
 	It("by digest with oci layout but without experimental", func() {
-		GeneralHost(BaseOptions(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f"
+		HostWithOCILayout(BaseOptions(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
 			expectedErrMsg := "Error: flag(s) --oci-layout in \"notation sign\" is experimental and not enabled by default. To use, please set NOTATION_EXPERIMENTAL=1 environment variable\n"
-			ociLayoutReference := OCILayoutTestPath + "@" + digest
-			notation.ExpectFailure().Exec("sign", "--oci-layout", ociLayoutReference).
+			notation.ExpectFailure().Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()).
 				MatchErrContent(expectedErrMsg)
 		})
 	})

test/e2e/suite/command/verify.go

@@ -71,52 +71,45 @@ var _ = Describe("notation verify", func() {
 	})
 
 	It("by digest with oci layout", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f"
-			ociLayoutReference := OCILayoutTestPath + "@" + digest
-			notation.Exec("sign", "--oci-layout", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(SignSuccessfully)
 
 			experimentalMsg := "Warning: This feature is experimental and may not be fully tested or completed and may be deprecated. Report any issues to \"https://github/notaryproject/notation\"\n"
-			notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayoutReference).
+			notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(VerifySuccessfully).
 				MatchErrKeyWords(experimentalMsg)
 		})
 	})
 
 	It("by tag with oci layout and COSE format", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			ociLayoutReference := OCILayoutTestPath + ":" + TestTag
-			notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", "--signature-format", "cose", ociLayout.ReferenceWithTag()).
 				MatchKeyWords(SignSuccessfully)
 
 			experimentalMsg := "Warning: This feature is experimental and may not be fully tested or completed and may be deprecated. Report any issues to \"https://github/notaryproject/notation\"\n"
-			notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayoutReference).
+			notation.Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayout.ReferenceWithTag()).
 				MatchKeyWords(VerifySuccessfully).
 				MatchErrKeyWords(experimentalMsg)
 		})
 	})
 
 	It("by digest with oci layout but without experimental", func() {
-		GeneralHost(BaseOptions(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f"
+		HostWithOCILayout(BaseOptions(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
 			expectedErrMsg := "Error: flag(s) --oci-layout,--scope in \"notation verify\" is experimental and not enabled by default. To use, please set NOTATION_EXPERIMENTAL=1 environment variable\n"
-			ociLayoutReference := OCILayoutTestPath + "@" + digest
-			notation.ExpectFailure().Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayoutReference).
+			notation.ExpectFailure().Exec("verify", "--oci-layout", "--scope", "local/e2e", ociLayout.ReferenceWithDigest()).
 				MatchErrContent(expectedErrMsg)
 		})
 	})
 
 	It("by digest with oci layout but missing scope", func() {
-		GeneralHost(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, vhost *utils.VirtualHost) {
-			const digest = "sha256:cc2ae4e91a31a77086edbdbf4711de48e5fa3ebdacad3403e61777a9e1a53b6f"
-			ociLayoutReference := OCILayoutTestPath + "@" + digest
-			notation.Exec("sign", "--oci-layout", ociLayoutReference).
+		HostWithOCILayout(BaseOptionsWithExperimental(), func(notation *utils.ExecOpts, ociLayout *OCILayout, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "--oci-layout", ociLayout.ReferenceWithDigest()).
 				MatchKeyWords(SignSuccessfully)
 
 			experimentalMsg := "Warning: This feature is experimental and may not be fully tested or completed and may be deprecated. Report any issues to \"https://github/notaryproject/notation\"\n"
 			expectedErrMsg := "Error: if any flags in the group [oci-layout scope] are set they must all be set; missing [scope]"
-			notation.ExpectFailure().Exec("verify", "--oci-layout", ociLayoutReference).
+			notation.ExpectFailure().Exec("verify", "--oci-layout", ociLayout.ReferenceWithDigest()).
 				MatchErrKeyWords(experimentalMsg).
 				MatchErrKeyWords(expectedErrMsg)
 		})
@@ -131,8 +124,8 @@ var _ = Describe("notation verify", func() {
 				MatchKeyWords(
 					VerifySuccessfully,
 				).
-				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
-				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+				MatchErrKeyWords(HTTPSRequest).
+				NoMatchErrKeyWords(HTTPRequest)
 		})
 	})