Adds a flag to skip conformance tests against non-required API (#11769)

* add flag

* use flag to guide imagepullpolicy

* comment change

* fix lint

* skip as well

* change flag name

* skip multiple containers

* skip generateName tests

test/conformance/api/v1/configuration_test.go

@@ -31,6 +31,10 @@ import (
 )
 
 func TestUpdateConfigurationMetadata(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Configuration create/patch/replace APIs are not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 

test/conformance/api/v1/generatename_test.go

@@ -114,6 +114,9 @@ func canServeRequests(t *testing.T, clients *test.Clients, route *v1.Route) erro
 // the system using metadata.generateName. It ensures that knative Services created this way can become ready
 // and serve requests.
 func TestServiceGenerateName(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Metadata.generateName is not required by Knative Serving API Specification")
+	}
 	t.Parallel()
 	clients := test.Setup(t)
 
@@ -150,6 +153,9 @@ func TestServiceGenerateName(t *testing.T) {
 // 1. Become ready
 // 2. Can serve requests.
 func TestRouteAndConfigGenerateName(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Metadata.generateName is not required by Knative Serving API Specification")
+	}
 	t.Parallel()
 	clients := test.Setup(t)
 

test/conformance/api/v1/route_test.go

@@ -97,6 +97,10 @@ func getRouteURL(clients *test.Clients, names test.ResourceNames) (*url.URL, err
 }
 
 func TestRouteCreation(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Route create/patch/replace APIs are not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 

test/conformance/api/v1/service_account_test.go

@@ -34,6 +34,10 @@ const (
 )
 
 func TestServiceAccountValidation(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.serviceAccountName is not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 

test/conformance/api/v1/service_test.go

@@ -624,6 +624,9 @@ func TestAnnotationPropagation(t *testing.T) {
 // TestServiceCreateWithMultipleContainers tests both Creation paths for a service.
 // The test performs a series of Validate steps to ensure that the service transitions as expected during each step.
 func TestServiceCreateWithMultipleContainers(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Multiple containers support is not required by Knative Serving API Specification")
+	}
 	if !test.ServingFlags.EnableBetaFeatures {
 		t.Skip()
 	}

test/conformance/api/v1/volumes_test.go

@@ -40,6 +40,10 @@ import (
 
 // TestConfigMapVolume tests that we echo back the appropriate text from the ConfigMap volume.
 func TestConfigMapVolume(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.volumes is not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 
@@ -107,6 +111,10 @@ func TestConfigMapVolume(t *testing.T) {
 
 // TestProjectedConfigMapVolume tests that we echo back the appropriate text from the ConfigMap volume.
 func TestProjectedConfigMapVolume(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.volumes is not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 
@@ -177,6 +185,9 @@ func TestProjectedConfigMapVolume(t *testing.T) {
 // TestSecretVolume tests that we echo back the appropriate text from the Secret volume.
 func TestSecretVolume(t *testing.T) {
 	t.Parallel()
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.volumes is not required by Knative Serving API Specification")
+	}
 	clients := test.Setup(t)
 
 	names := test.ResourceNames{
@@ -240,6 +251,9 @@ func TestSecretVolume(t *testing.T) {
 // TestProjectedSecretVolume tests that we echo back the appropriate text from the Secret volume.
 func TestProjectedSecretVolume(t *testing.T) {
 	t.Parallel()
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.volumes is not required by Knative Serving API Specification")
+	}
 	clients := test.Setup(t)
 
 	names := test.ResourceNames{
@@ -307,6 +321,9 @@ func TestProjectedSecretVolume(t *testing.T) {
 // TestProjectedComplex tests that we echo back the appropriate text from the complex Projected volume.
 func TestProjectedComplex(t *testing.T) {
 	t.Parallel()
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.volumes is not required by Knative Serving API Specification")
+	}
 	clients := test.Setup(t)
 
 	names := test.ResourceNames{
@@ -410,6 +427,9 @@ func TestProjectedComplex(t *testing.T) {
 // TestProjectedServiceAccountToken tests that a valid JWT service account token can be mounted.
 func TestProjectedServiceAccountToken(t *testing.T) {
 	t.Parallel()
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Service.spec.volumes is not required by Knative Serving API Specification")
+	}
 	clients := test.Setup(t)
 
 	names := test.ResourceNames{

test/conformance/runtime/envpropagation_test.go

@@ -53,6 +53,9 @@ func TestSecretsViaEnv(t *testing.T) {
 	})
 
 	t.Run("envFrom", func(t *testing.T) {
+		if test.ServingFlags.DisableOptionalAPI {
+			t.Skip("Container.envFrom is not required by Knative Serving API Specification")
+		}
 		t.Parallel()
 
 		err := fetchEnvironmentAndVerify(t, clients, WithEnvFrom(corev1.EnvFromSource{
@@ -93,6 +96,9 @@ func TestConfigsViaEnv(t *testing.T) {
 	})
 
 	t.Run("envFrom", func(t *testing.T) {
+		if test.ServingFlags.DisableOptionalAPI {
+			t.Skip("Container.envFrom is not required by Knative Serving API Specification")
+		}
 		t.Parallel()
 
 		err := fetchEnvironmentAndVerify(t, clients, WithEnvFrom(corev1.EnvFromSource{

test/conformance/runtime/readiness_probe_test.go

@@ -45,6 +45,9 @@ const readinessPropagationTime = 30 * time.Second
 
 func TestProbeRuntime(t *testing.T) {
 	t.Parallel()
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Container.readinessProbe is not required by Knative Serving API Specification")
+	}
 	clients := test.Setup(t)
 
 	var testCases = []struct {

test/conformance/runtime/user_test.go

@@ -35,6 +35,10 @@ const (
 // TestMustRunAsUser verifies that a supplied runAsUser through securityContext takes
 // effect as declared by "MUST" in the runtime-contract.
 func TestMustRunAsUser(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Container.securityContext is not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 
@@ -73,6 +77,10 @@ func TestMustRunAsUser(t *testing.T) {
 // in the Dockerfile is respected when executed in Knative as declared by "SHOULD"
 // in the runtime-contract.
 func TestShouldRunAsUserContainerDefault(t *testing.T) {
+	if test.ServingFlags.DisableOptionalAPI {
+		t.Skip("Container.securityContext is not required by Knative Serving API Specification")
+	}
+
 	t.Parallel()
 	clients := test.Setup(t)
 	_, ri, err := fetchRuntimeInfo(t, clients)

test/e2e_flags.go

@@ -39,6 +39,7 @@ type ServingEnvironmentFlags struct {
 	EnableAlphaFeatures bool   // Indicates whether we run tests for alpha features
 	EnableBetaFeatures  bool   // Indicates whether we run tests for beta features
 	DisableLogStream    bool   // Indicates whether log streaming is disabled
+	DisableOptionalAPI  bool   // Indicates whether to skip conformance tests against optional API
 	TestNamespace       string // Default namespace for Serving E2E/Conformance tests
 	AltTestNamespace    string // Alternative namespace for running cross-namespace tests in
 	TLSTestNamespace    string // Namespace for Serving TLS tests
@@ -71,6 +72,9 @@ func initializeServingFlags() *ServingEnvironmentFlags {
 	flag.BoolVar(&f.DisableLogStream, "disable-logstream", false,
 		"Set this flag to disable streaming logs from system components")
 
+	flag.BoolVar(&f.DisableOptionalAPI, "disable-optional-api", false,
+		"Set this flag to skip conformance tests against optional API.")
+
 	flag.StringVar(&f.TestNamespace, "test-namespace", "serving-tests",
 		"Set this flag to change the default namespace for running tests.")
 

test/v1/configuration.go

@@ -107,24 +107,32 @@ func WaitForConfigLatestRevision(clients *test.Clients, names test.ResourceNames
 // ConfigurationSpec returns the spec of a configuration to be used throughout different
 // CRD helpers.
 func ConfigurationSpec(imagePath string) *v1.ConfigurationSpec {
-	return &v1.ConfigurationSpec{
+	c := &v1.ConfigurationSpec{
 		Template: v1.RevisionTemplateSpec{
 			Spec: v1.RevisionSpec{
 				PodSpec: corev1.PodSpec{
 					Containers: []corev1.Container{{
 						Image: imagePath,
-						// Kubernetes default pull policy is IfNotPresent unless
-						// the :latest tag (== no tag) is used, in which case it
-						// is Always.  To support e2e testing on KinD, we want to
-						// explicitly disable image pulls when present because we
-						// side-load the test images onto all nodes and never push
-						// them to a registry.
-						ImagePullPolicy: corev1.PullIfNotPresent,
 					}},
 				},
 			},
 		},
 	}
+
+	if !test.ServingFlags.DisableOptionalAPI {
+		// Container.imagePullPolicy is not required by Knative
+		// Serving API Specification.
+		//
+		// Kubernetes default pull policy is IfNotPresent unless
+		// the :latest tag (== no tag) is used, in which case it
+		// is Always.  To support e2e testing on KinD, we want to
+		// explicitly disable image pulls when present because we
+		// side-load the test images onto all nodes and never push
+		// them to a registry.
+		c.Template.Spec.PodSpec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent
+	}
+
+	return c
 }
 
 // Configuration returns a Configuration object in namespace with the name names.Config