Improve manifest stack application w.r.t. CRDs

inttest/Makefile.variables

@@ -59,5 +59,6 @@ smoketests := \
 	check-psp \
 	check-reset \
 	check-singlenode \
+	check-stackapplier \
 	check-statussocket \
 	check-upgrade \

inttest/common/autopilot/waitfor.go

@@ -56,16 +56,10 @@ func WaitForPlanState(ctx context.Context, client apclient.Interface, name strin
 
 // WaitForCRDByName waits until the CRD with the given name is established.
 func WaitForCRDByName(ctx context.Context, client extensionsclient.ApiextensionsV1Interface, name string) error {
-	// Some shortcuts for very long type names.
-	type (
-		crd     = extensionsv1.CustomResourceDefinition
-		crdList = extensionsv1.CustomResourceDefinitionList
-	)
-
-	return watch.FromClient[*crdList, crd](client.CustomResourceDefinitions()).
+	return watch.CRDs(client.CustomResourceDefinitions()).
 		WithObjectName(fmt.Sprintf("%s.%s", name, apv1beta2.GroupName)).
 		WithErrorCallback(common.RetryWatchErrors(logrus.Infof)).
-		Until(ctx, func(item *crd) (bool, error) {
+		Until(ctx, func(item *extensionsv1.CustomResourceDefinition) (bool, error) {
 			for _, cond := range item.Status.Conditions {
 				if cond.Type == extensionsv1.Established {
 					return cond.Status == extensionsv1.ConditionTrue, nil

inttest/stackapplier/rings.yaml

@@ -0,0 +1,82 @@
+# Have the wrong order: First the custom resources, then their definitions. Let
+# one of the the resource be a cluster resource, so that the Stack's resource
+# reordering won't let the CRD go before the CR.
+
+---
+apiVersion: k0s.example.com/v1
+kind: Character
+metadata:
+  name: frodo
+  namespace: shire
+spec:
+  speciesRef:
+    name: hobbit
+
+---
+apiVersion: k0s.example.com/v1
+kind: Species
+metadata:
+  name: hobbit
+spec:
+  characteristics: hairy feet
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: shire
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: species.k0s.example.com
+spec:
+  group: k0s.example.com
+  names:
+    kind: Species
+    singular: species
+    plural: species
+  scope: Cluster
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                characteristics:
+                  type: string
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: characters.k0s.example.com
+spec:
+  group: k0s.example.com
+  names:
+    kind: Character
+    singular: character
+    plural: characters
+  scope: Namespaced
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          type: object
+          properties:
+            spec:
+              type: object
+              properties:
+                speciesRef:
+                  type: object
+                  properties:
+                    name:
+                      type: string

inttest/stackapplier/stackapplier_test.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2024 k0s authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package nllb
+
+import (
+	"context"
+	_ "embed"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/k0sproject/k0s/inttest/common"
+	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	"github.com/k0sproject/k0s/pkg/kubernetes/watch"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/dynamic"
+
+	"github.com/stretchr/testify/assert"
+	testifysuite "github.com/stretchr/testify/suite"
+	"sigs.k8s.io/yaml"
+)
+
+type suite struct {
+	common.BootlooseSuite
+}
+
+//go:embed rings.yaml
+var rings string
+
+func (s *suite) TestStackApplier() {
+	ctx, cancel := context.WithCancelCause(s.Context())
+	s.T().Cleanup(func() { cancel(nil) })
+
+	k0sConfig, err := yaml.Marshal(&v1beta1.ClusterConfig{
+		Spec: &v1beta1.ClusterSpec{
+			Storage: &v1beta1.StorageSpec{Type: v1beta1.KineStorageType},
+		},
+	})
+	s.Require().NoError(err)
+
+	s.WriteFileContent(s.ControllerNode(0), "/tmp/k0s.yaml", k0sConfig)
+	s.Require().NoError(s.InitController(0, "--config=/tmp/k0s.yaml", "--disable-components=control-api,konnectivity-server,kube-controller-manager,kube-scheduler"))
+	s.MakeDir(s.ControllerNode(0), "/var/lib/k0s/manifests/rings")
+	s.PutFile(s.ControllerNode(0), "/var/lib/k0s/manifests/rings/rings.yaml", rings)
+
+	kubeconfig, err := s.GetKubeConfig(s.ControllerNode(0))
+	s.Require().NoError(err)
+	client, err := dynamic.NewForConfig(kubeconfig)
+	s.Require().NoError(err)
+
+	sgv := schema.GroupVersion{Group: "k0s.example.com", Version: "v1"}
+
+	s.T().Run("hobbit", func(t *testing.T) {
+		t.Cleanup(func() {
+			if t.Failed() {
+				cancel(fmt.Errorf("%s failed", t.Name()))
+			}
+		})
+		t.Parallel()
+		species := client.Resource(sgv.WithResource("species"))
+		assert.NoError(t, watch.Unstructured(species).
+			WithObjectName("hobbit").
+			WithErrorCallback(retryWatchErrors(s.T().Logf)).
+			Until(ctx, func(item *unstructured.Unstructured) (bool, error) {
+				speciesName, found, err := unstructured.NestedString(item.Object, "spec", "characteristics")
+				if assert.NoError(t, err) && assert.True(t, found, "no characteristics found: %v", item.Object) {
+					assert.Equal(t, "hairy feet", speciesName)
+				}
+				return true, nil
+			}))
+	})
+
+	s.T().Run("frodo", func(t *testing.T) {
+		t.Cleanup(func() {
+			if t.Failed() {
+				cancel(fmt.Errorf("%s failed", t.Name()))
+			}
+		})
+		t.Parallel()
+		characters := client.Resource(sgv.WithResource("characters"))
+		assert.NoError(t, watch.Unstructured(characters.Namespace("shire")).
+			WithObjectName("frodo").
+			WithErrorCallback(retryWatchErrors(s.T().Logf)).
+			Until(ctx, func(item *unstructured.Unstructured) (bool, error) {
+				speciesName, found, err := unstructured.NestedString(item.Object, "spec", "speciesRef", "name")
+				if assert.NoError(t, err) && assert.True(t, found, "no species found: %v", item.Object) {
+					assert.Equal(t, "hobbit", speciesName)
+				}
+				return true, nil
+			}))
+	})
+}
+
+func retryWatchErrors(logf common.LogfFn) watch.ErrorCallback {
+	commonRetry := common.RetryWatchErrors(logf)
+	return func(err error) (time.Duration, error) {
+		if retryDelay, err := commonRetry(err); err == nil {
+			return retryDelay, nil
+		}
+		if apierrors.IsNotFound(err) {
+			return 350 * time.Millisecond, nil
+		}
+		return 0, err
+	}
+}
+
+func TestStackApplierSuite(t *testing.T) {
+	s := suite{
+		common.BootlooseSuite{
+			ControllerCount: 1,
+			WorkerCount:     0,
+		},
+	}
+	testifysuite.Run(t, &s)
+}

pkg/applier/applier.go

@@ -25,8 +25,6 @@ import (
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/cli-runtime/pkg/resource"
-	"k8s.io/client-go/discovery"
-	"k8s.io/client-go/dynamic"
 
 	"github.com/sirupsen/logrus"
 )
@@ -43,10 +41,8 @@ type Applier struct {
 	Name string
 	Dir  string
 
-	log             *logrus.Entry
-	clientFactory   kubernetes.ClientFactoryInterface
-	client          dynamic.Interface
-	discoveryClient discovery.CachedDiscoveryInterface
+	log           *logrus.Entry
+	clientFactory kubernetes.ClientFactoryInterface
 
 	restClientGetter resource.RESTClientGetter
 }
@@ -70,35 +66,8 @@ func NewApplier(dir string, kubeClientFactory kubernetes.ClientFactoryInterface)
 	}
 }
 
-func (a *Applier) lazyInit() error {
-	if a.client == nil {
-		c, err := a.clientFactory.GetDynamicClient()
-		if err != nil {
-			return err
-		}
-
-		a.client = c
-	}
-
-	if a.discoveryClient == nil {
-		c, err := a.clientFactory.GetDiscoveryClient()
-		if err != nil {
-			return err
-		}
-
-		a.discoveryClient = c
-	}
-
-	return nil
-}
-
 // Apply resources
 func (a *Applier) Apply(ctx context.Context) error {
-	err := a.lazyInit()
-	if err != nil {
-		return err
-	}
-
 	files, err := FindManifestFilesInDir(a.Dir)
 	if err != nil {
 		return err
@@ -111,14 +80,12 @@ func (a *Applier) Apply(ctx context.Context) error {
 	stack := Stack{
 		Name:      a.Name,
 		Resources: resources,
-		Client:    a.client,
-		Discovery: a.discoveryClient,
+		Clients:   a.clientFactory,
 	}
 	a.log.Debug("applying stack")
 	err = stack.Apply(ctx, true)
 	if err != nil {
 		a.log.WithError(err).Warn("stack apply failed")
-		a.discoveryClient.Invalidate()
 	} else {
 		a.log.Debug("successfully applied stack")
 	}
@@ -128,18 +95,9 @@ func (a *Applier) Apply(ctx context.Context) error {
 
 // Delete deletes the entire stack by applying it with empty set of resources
 func (a *Applier) Delete(ctx context.Context) error {
-	err := a.lazyInit()
-	if err != nil {
-		return err
-	}
-	stack := Stack{
-		Name:      a.Name,
-		Client:    a.client,
-		Discovery: a.discoveryClient,
-	}
+	stack := Stack{Name: a.Name, Clients: a.clientFactory}
 	logrus.Debugf("about to delete a stack %s with empty apply", a.Name)
-	err = stack.Apply(ctx, true)
-	return err
+	return stack.Apply(ctx, true)
 }
 
 func (a *Applier) parseFiles(files []string) ([]*unstructured.Unstructured, error) {
@@ -149,6 +107,7 @@ func (a *Applier) parseFiles(files []string) ([]*unstructured.Unstructured, erro
 	}
 
 	objects, err := resource.NewBuilder(a.restClientGetter).
+		Local(). // don't fail on unknown CRDs
 		Unstructured().
 		Path(false, files...).
 		Flatten().