test: add e2e test cases for flag --insecure-registry (notaryprojec…

…t#679) Feature: - add E2E testing TLS support for Github Action environment Test: - added `--insecure-registry` test for sign, verify, inspect, list > **Note** `login` command E2E test cases are not included in the PR because it needs to setup credential store in Github Action and should be done in another PR. Added an issue to track it notaryproject#680 Resolves notaryproject#633 Signed-off-by: Junjie Gao <[email protected]> --------- Signed-off-by: Junjie Gao <[email protected]>
Wwwsylvia · May 25, 2023 · 98f5290 · 98f5290
1 parent ed22fde
commit 98f5290
Show file tree

Hide file tree

Showing 16 changed files with 376 additions and 12 deletions.
diff --git a/test/e2e/internal/notation/host.go b/test/e2e/internal/notation/host.go
@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/notaryproject/notation/test/e2e/internal/utils"
+	. "github.com/onsi/ginkgo/v2"
 )
 
 // CoreTestFunc is the test function running in a VirtualHost.
@@ -40,6 +41,17 @@ func Host(options []utils.HostOption, fn CoreTestFunc) {
 	fn(vhost.Executor, artifact, vhost)
 }
 
+// HostInGithubAction only run the test in GitHub Actions.
+//
+// The booting script will setup TLS reverse proxy and TLS certificate
+// for Github Actions environment.
+func HostInGithubAction(options []utils.HostOption, fn CoreTestFunc) {
+	if os.Getenv("GITHUB_ACTIONS") != "true" {
+		Skip("only run in GitHub Actions")
+	}
+	Host(options, fn)
+}
+
 // GeneralHost creates a virtualized notation testing host by modify
 // the "XDG_CONFIG_HOME" environment variable of the Executor. It's agnostic to
 // the Repository of the artifact.

diff --git a/test/e2e/internal/notation/init.go b/test/e2e/internal/notation/init.go
@@ -23,6 +23,7 @@ const (
 	envKeyRegistryHost       = "NOTATION_E2E_REGISTRY_HOST"
 	envKeyRegistryUsername   = "NOTATION_E2E_REGISTRY_USERNAME"
 	envKeyRegistryPassword   = "NOTATION_E2E_REGISTRY_PASSWORD"
+	envKeyDomainRegistryHost = "NOTATION_E2E_DOMAIN_REGISTRY_HOST"
 	envKeyNotationBinPath    = "NOTATION_E2E_BINARY_PATH"
 	envKeyNotationOldBinPath = "NOTATION_E2E_OLD_BINARY_PATH"
 	envKeyNotationPluginPath = "NOTATION_E2E_PLUGIN_PATH"
@@ -65,6 +66,7 @@ func setUpRegistry() {
 	setValue(envKeyRegistryHost, &TestRegistry.Host)
 	setValue(envKeyRegistryUsername, &TestRegistry.Username)
 	setValue(envKeyRegistryPassword, &TestRegistry.Password)
+	setValue(envKeyDomainRegistryHost, &TestRegistry.DomainHost)
 
 	setPathValue(envKeyOCILayoutPath, &OCILayoutPath)
 	setValue(envKeyTestRepo, &TestRepoUri)

diff --git a/test/e2e/internal/notation/registry.go b/test/e2e/internal/notation/registry.go
@@ -19,9 +19,19 @@ import (
 const ArtifactTypeNotation = "application/vnd.cncf.notary.signature"
 
 type Registry struct {
-	Host     string
+	// Host is the registry host.
+	Host string
+	// Username is the username to access the registry.
 	Username string
+	// Password is the password to access the registry.
 	Password string
+	// DomainHost is a registry host, separate from localhost, used for testing
+	// the --insecure-registry flag.
+	//
+	// If the host is localhost, Notation connects via plain HTTP. For
+	// non-localhost hosts, Notation defaults to HTTPS. However, users can
+	// enforce HTTP by setting the --insecure-registry flag.
+	DomainHost string
 }
 
 // CreateArtifact copies a local OCI layout to the registry to create
@@ -101,6 +111,12 @@ func (r *Artifact) ReferenceWithDigest() string {
 	return fmt.Sprintf("%s/%s@%s", r.Host, r.Repo, r.Digest)
 }
 
+// DomainReferenceWithDigest returns the <domainHost>/<Repository>@<alg>:<digest>
+// for testing --insecure-registry flag and TLS request.
+func (r *Artifact) DomainReferenceWithDigest() string {
+	return fmt.Sprintf("%s/%s@%s", r.DomainHost, r.Repo, r.Digest)
+}
+
 // SignatureManifest returns the manifest of the artifact.
 func (r *Artifact) SignatureDescriptors() ([]ocispec.Descriptor, error) {
 	ctx := context.Background()

diff --git a/test/e2e/internal/utils/matcher.go b/test/e2e/internal/utils/matcher.go
@@ -5,9 +5,15 @@ import (
 	"strings"
 
 	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
 	"github.com/onsi/gomega/gexec"
 )
 
+func init() {
+	// expand the length limit for the gomega matcher
+	format.MaxLength = 1000000
+}
+
 // Matcher contains the execution result for matching.
 type Matcher struct {
 	Session *gexec.Session
@@ -48,6 +54,15 @@ func (m *Matcher) MatchErrKeyWords(keywords ...string) *Matcher {
 	return m
 }
 
+// NoMatchErrKeyWords guarantees that the given keywords do not match with
+// the stderr.
+func (m *Matcher) NoMatchErrKeyWords(keywords ...string) *Matcher {
+	for _, w := range keywords {
+		Expect(m.stderr).ShouldNot(ContainSubstring(w))
+	}
+	return m
+}
+
 // MatchErrKeyWords matches given keywords with the stderr.
 func matchKeyWords(content string, keywords []string) {
 	var missed []string

diff --git a/test/e2e/run.sh b/test/e2e/run.sh
@@ -40,7 +40,7 @@ if [ ! -f "$NOTATION_E2E_OLD_BINARY_PATH" ]; then
     echo "Try to use old notation binary at $NOTATION_E2E_OLD_BINARY_PATH"
 
     if [ ! -f $NOTATION_E2E_OLD_BINARY_PATH ]; then
-        TAG=1.0.0-rc.2 # without 'v'
+        TAG=1.0.0-rc.5 # without 'v'
         echo "Didn't find old notation binary locally. Try to download notation v$TAG."
 
         TAR_NAME=notation_${TAG}_linux_amd64.tar.gz

diff --git a/test/e2e/scripts/dockerhub.sh b/test/e2e/scripts/dockerhub.sh
@@ -16,9 +16,10 @@ if [ -z "$DOCKER_USERNAME" ] || [ -z "$DOCKER_PASSWORD" ]; then
 fi
 
 # set environment variables for E2E testing
-export NOTATION_E2E_REGISTRY_HOST=docker.io/$DOCKER_USERNAME
-export NOTATION_E2E_REGISTRY_USERNAME=$DOCKER_USERNAME
-export NOTATION_E2E_REGISTRY_PASSWORD=$DOCKER_PASSWORD
+export NOTATION_E2E_REGISTRY_HOST="docker.io/$DOCKER_USERNAME"
+export NOTATION_E2E_REGISTRY_USERNAME="$DOCKER_USERNAME"
+export NOTATION_E2E_REGISTRY_PASSWORD="$DOCKER_PASSWORD"
+export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$NOTATION_E2E_REGISTRY_HOST"
 
 function setup_registry {
     echo "use $NOTATION_E2E_REGISTRY_HOST"
@@ -55,4 +56,4 @@ function cleanup_registry {
                 echo "$NOTATION_E2E_REGISTRY_HOST/$repoName deleted."
         done
     done
-}
+}
diff --git a/test/e2e/scripts/tls.sh b/test/e2e/scripts/tls.sh
@@ -0,0 +1,51 @@
+#!/bin/bash -e
+#
+# Usage
+#   For setup:
+#   1. source ./scripts/tls.sh
+#   2. call create_docker_network
+#   3. setup registry with port 5000 in $DOCKER_NETWORK
+#   4. call setup_tls reverse proxy
+#
+#   For clean up:
+#   1. call clean_up
+#   2. clean up registry
+#   3. call remove_docker_network
+# 
+# Note: this script needs sudo permission to add TLS certificate to system and 
+#       add domain registry host.
+
+NGINX_CONTAINER_NAME=nginx
+DOMAIN=notation-e2e.registry.io
+DOCKER_NETWORK=notation-e2e
+
+function create_docker_network {
+    docker network create "$DOCKER_NETWORK"
+}
+
+function remove_docker_network {
+    docker network rm "$DOCKER_NETWORK"
+}
+
+function setup_tls {
+    # add domain registry host to /etc/hosts for testing --plain-http feature
+    echo "127.0.0.1 $DOMAIN" | sudo tee -a /etc/hosts
+    # add TLS certificate to system
+    sudo mkdir -p /usr/local/share/ca-certificates/
+    sudo cp ./testdata/nginx/notation-e2e.registry.io.crt /usr/local/share/ca-certificates/
+    sudo update-ca-certificates
+
+    # start Nginx for TLS 
+    docker run -d -p 80:80 -p 443:443 \
+        --network "$DOCKER_NETWORK" \
+        --mount type=bind,source="$(pwd)/testdata/nginx/",target=/etc/nginx \
+        --name "$NGINX_CONTAINER_NAME" \
+        --rm nginx:latest
+}
+
+function clean_up_tls {
+    docker container stop "$NGINX_CONTAINER_NAME" 1>/dev/null && echo "Nginx stopped"
+    sudo sed -i "/${NOTATION_E2E_DOMAIN_REGISTRY_HOST}/d" /etc/hosts
+    sudo rm /usr/local/share/ca-certificates/notation-e2e.registry.io.crt
+    sudo update-ca-certificates
+}
diff --git a/test/e2e/scripts/zot.sh b/test/e2e/scripts/zot.sh
@@ -4,24 +4,38 @@
 # Usage
 #   ./run.sh zot <notation-binary-path> [old-notation-binary-path]
 
+source ./scripts/tls.sh
+
 REG_HOST=localhost
 REG_PORT=5000
-ZOT_CONTAINER_NAME=zot
+ZOT_CONTAINER_NAME=notation-e2e-registry
 
-# set environment variables for E2E testing
-export NOTATION_E2E_REGISTRY_HOST=$REG_HOST:$REG_PORT
+# set required environment variables for E2E testing
+export NOTATION_E2E_REGISTRY_HOST="$REG_HOST:$REG_PORT"
 export NOTATION_E2E_REGISTRY_USERNAME=testuser
 export NOTATION_E2E_REGISTRY_PASSWORD=testpassword
+export NOTATION_E2E_DOMAIN_REGISTRY_HOST="$DOMAIN"
 
 function setup_registry {
+    create_docker_network
     # start Zot
-    docker run -d -p $REG_PORT:$REG_PORT -it --name $ZOT_CONTAINER_NAME \
-        --mount type=bind,source=`pwd`/testdata/registry/zot/,target=/etc/zot \
+    docker run -d -p $REG_PORT:$REG_PORT -it \
+        --name $ZOT_CONTAINER_NAME \
+        --network $DOCKER_NETWORK \
+        --mount type=bind,source="$(pwd)/testdata/registry/zot/",target=/etc/zot \
         --rm ghcr.io/project-zot/zot-minimal-linux-amd64:latest
+
+    if [ "$GITHUB_ACTIONS" == "true" ]; then
+        setup_tls
+    fi
     # make sure that Zot is ready
     sleep 1
 }
 
 function cleanup_registry {
     docker container stop $ZOT_CONTAINER_NAME 1>/dev/null && echo "Zot stopped"
-}
+    if [ "$GITHUB_ACTIONS" == "true" ]; then
+        clean_up_tls
+    fi
+    remove_docker_network
+}
diff --git a/test/e2e/suite/command/inspect.go b/test/e2e/suite/command/inspect.go
@@ -0,0 +1,67 @@
+package command
+
+import (
+	. "github.com/notaryproject/notation/test/e2e/internal/notation"
+	"github.com/notaryproject/notation/test/e2e/internal/utils"
+	. "github.com/notaryproject/notation/test/e2e/suite/common"
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var inspectSuccessfully = []string{
+	"└── application/vnd.cncf.notary.signature",
+	"└── sha256:",
+	"├── media type:",
+	"├── signature algorithm:",
+	"├── signed attributes",
+	"signingTime:",
+	"signingScheme:",
+	"├── user defined attributes",
+	"│   └── (empty)",
+	"├── unsigned attributes",
+	"│   └── signingAgent: Notation/",
+	"├── certificates",
+	"│   └── SHA256 fingerprint:",
+	"issued to:",
+	"issued by:",
+	"expiry:",
+	"└── signed artifact",
+	"media type:",
+	"digest:",
+	"size:",
+}
+
+var _ = Describe("notation inspect", func() {
+	It("all signatures of an image", func() {
+		Host(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.ReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("inspect", "-d", artifact.ReferenceWithDigest()).
+				MatchKeyWords(inspectSuccessfully...)
+		})
+	})
+
+	It("all signatures of an image with TLS", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("inspect", "-d", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(inspectSuccessfully...).
+				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
+				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+		})
+	})
+
+	It("all signatures of an image with --insecure-registry flag", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("inspect", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(inspectSuccessfully...).
+				MatchErrKeyWords(HTTPRequest).
+				NoMatchErrKeyWords(HTTPSRequest)
+		})
+	})
+})
diff --git a/test/e2e/suite/command/list.go b/test/e2e/suite/command/list.go
@@ -0,0 +1,53 @@
+package command
+
+import (
+	. "github.com/notaryproject/notation/test/e2e/internal/notation"
+	"github.com/notaryproject/notation/test/e2e/internal/utils"
+	. "github.com/notaryproject/notation/test/e2e/suite/common"
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("notation list", func() {
+	It("all signatures of an image", func() {
+		Host(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.ReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("list", "-d", artifact.ReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:",
+				)
+		})
+	})
+
+	It("all signatures of an image with TLS", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("list", "-d", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:",
+				).
+				MatchErrKeyWords("https://notation-e2e.registry.io/v2/e2e").
+				NoMatchErrKeyWords("http://notation-e2e.registry.io")
+		})
+	})
+
+	It("all signatures of an image with --insecure-registry flag", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully)
+
+			notation.Exec("list", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(
+					"└── application/vnd.cncf.notary.signature",
+					"└── sha256:",
+				).
+				MatchErrKeyWords(HTTPRequest).
+				NoMatchErrKeyWords(HTTPSRequest)
+		})
+	})
+})
diff --git a/test/e2e/suite/command/sign.go b/test/e2e/suite/command/sign.go
@@ -150,4 +150,28 @@ var _ = Describe("notation sign", func() {
 				MatchErrContent(expectedErrMsg)
 		})
 	})
+
+	It("with TLS by digest", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "-d", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully).
+				MatchErrKeyWords(HTTPSRequest).
+				NoMatchErrKeyWords(HTTPRequest)
+
+			OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(VerifySuccessfully)
+		})
+	})
+
+	It("with --insecure-registry by digest", func() {
+		HostInGithubAction(BaseOptions(), func(notation *utils.ExecOpts, artifact *Artifact, vhost *utils.VirtualHost) {
+			notation.Exec("sign", "-d", "--insecure-registry", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(SignSuccessfully).
+				MatchErrKeyWords(HTTPRequest).
+				NoMatchErrKeyWords(HTTPSRequest)
+
+			OldNotation().Exec("verify", artifact.DomainReferenceWithDigest()).
+				MatchKeyWords(VerifySuccessfully)
+		})
+	})
 })