Version check between compiled syncer and yaml

Signed-off-by: David Festal <[email protected]>

cmd/syncer/cmd/syncer.go

@@ -116,6 +116,7 @@ func Run(ctx context.Context, options *synceroptions.Options) error {
 			SyncTargetUID:                 options.SyncTargetUID,
 			DNSImage:                      options.DNSImage,
 			DownstreamNamespaceCleanDelay: options.DownstreamNamespaceCleanDelay,
+			ExpectedKCPVersion:            options.ExpectedKCPVersion,
 		},
 		numThreads,
 		options.APIImportPollInterval,

cmd/syncer/options/options.go

@@ -45,6 +45,7 @@ type Options struct {
 	SyncedResourceTypes           []string
 	DNSImage                      string
 	DownstreamNamespaceCleanDelay time.Duration
+	ExpectedKCPVersion            string
 
 	APIImportPollInterval time.Duration
 }
@@ -82,6 +83,7 @@ func (options *Options) AddFlags(fs *pflag.FlagSet) {
 		"Options are:\n"+strings.Join(kcpfeatures.KnownFeatures(), "\n")) // hide kube-only gates
 	fs.StringVar(&options.DNSImage, "dns-image", options.DNSImage, "kcp DNS server image.")
 	fs.DurationVar(&options.DownstreamNamespaceCleanDelay, "downstream-namespace-clean-delay", options.DownstreamNamespaceCleanDelay, "Time to wait before deleting of a downstream namespace.")
+	fs.StringVar(&options.ExpectedKCPVersion, "expectedKCPVersion", options.ExpectedKCPVersion, "The expected compiled KCP version of the syncer")
 
 	options.Logs.AddFlags(fs)
 }

pkg/apis/workload/v1alpha1/synctarget_types.go

@@ -183,6 +183,10 @@ const (
 
 	// ErrorHeartbeatMissedReason indicates that a heartbeat update was not received within the configured threshold.
 	ErrorHeartbeatMissedReason = "ErrorHeartbeat"
+
+	// VersionMismatchReason indicates that the compiled version of the Syncer is not the same as the expected version
+	// (set by the `kcp workload sync` CLI command).
+	VersionMismatchReason = "ErrorVersionMismatch"
 )
 
 func (in *SyncTarget) SetConditions(conditions conditionsv1alpha1.Conditions) {

pkg/cliplugins/workload/plugin/sync.go

@@ -48,6 +48,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/pkg/version"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"k8s.io/kube-openapi/pkg/util/sets"
@@ -100,6 +101,8 @@ type SyncOptions struct {
 	APIImportPollInterval time.Duration
 	// FeatureGates is used to configure which feature gates are enabled.
 	FeatureGates string
+	// DisableSyncerVersionCheck is used to disable the version check in the deployed syncer.
+	DisableSyncerVersionCheck bool
 }
 
 // NewSyncOptions returns a new SyncOptions.
@@ -135,6 +138,7 @@ func (o *SyncOptions) BindFlags(cmd *cobra.Command) {
 		"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
 			"Options are:\n"+strings.Join(kcpfeatures.KnownFeatures(), "\n")) // hide kube-only gates
 	cmd.Flags().DurationVar(&o.APIImportPollInterval, "api-import-poll-interval", o.APIImportPollInterval, "Polling interval for API import.")
+	cmd.Flags().BoolVar(&o.DisableSyncerVersionCheck, "disable-syncer-version-check", o.DisableSyncerVersionCheck, "Option to disable the version check in the deployed syncer.")
 }
 
 // Complete ensures all dynamically populated fields are initialized.
@@ -262,6 +266,10 @@ func (o *SyncOptions) Run(ctx context.Context) error {
 		APIImportPollIntervalString: o.APIImportPollInterval.String(),
 	}
 
+	if !o.DisableSyncerVersionCheck {
+		input.ExpectedKCPVersion = version.Get().GitVersion
+	}
+
 	resources, err := renderSyncerResources(input, syncerID, expectedResourcesForPermission.List())
 	if err != nil {
 		return err
@@ -696,6 +704,8 @@ type templateInput struct {
 	FeatureGatesString string
 	// APIImportPollIntervalString is the string of interval to poll APIImport.
 	APIImportPollIntervalString string
+	// ExpectedKCPVersion is the expected version of KCP for the syncer image
+	ExpectedKCPVersion string
 }
 
 // templateArgs represents the full set of arguments required to render the resources

pkg/cliplugins/workload/plugin/sync_test.go

@@ -180,6 +180,235 @@ stringData:
 ---
 apiVersion: apps/v1
 kind: Deployment
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: kcp-syncer-sync-target-name-34b23c4k
+  template:
+    metadata:
+      labels:
+        app: kcp-syncer-sync-target-name-34b23c4k
+    spec:
+      containers:
+      - name: kcp-syncer
+        command:
+        - /ko-app/syncer
+        args:
+        - --from-kubeconfig=/kcp/kubeconfig
+        - --sync-target-name=sync-target-name
+        - --sync-target-uid=sync-target-uid
+        - --from-cluster=root:default:foo
+        - --api-import-poll-interval=1m
+        - --resources=resource1
+        - --resources=resource2
+        - --qps=123.4
+        - --burst=456
+        - --dns-image=image
+        - --expectedKCPVersion=the-kcp-version
+        env:
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        image: image
+        imagePullPolicy: IfNotPresent
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - name: kcp-config
+          mountPath: /kcp/
+          readOnly: true
+      serviceAccountName: kcp-syncer-sync-target-name-34b23c4k
+      volumes:
+        - name: kcp-config
+          secret:
+            secretName: kcp-syncer-sync-target-name-34b23c4k
+            optional: false
+`
+
+	actualYAML, err := renderSyncerResources(templateInput{
+		ServerURL:                   "server-url",
+		Token:                       "token",
+		CAData:                      "ca-data",
+		KCPNamespace:                "kcp-namespace",
+		Namespace:                   "kcp-syncer-sync-target-name-34b23c4k",
+		LogicalCluster:              "root:default:foo",
+		SyncTarget:                  "sync-target-name",
+		SyncTargetUID:               "sync-target-uid",
+		Image:                       "image",
+		Replicas:                    1,
+		ResourcesToSync:             []string{"resource1", "resource2"},
+		APIImportPollIntervalString: "1m",
+		QPS:                         123.4,
+		Burst:                       456,
+		ExpectedKCPVersion:          "the-kcp-version",
+	}, "kcp-syncer-sync-target-name-34b23c4k", []string{"resource1", "resource2"})
+	require.NoError(t, err)
+	require.Empty(t, cmp.Diff(expectedYAML, string(actualYAML)))
+}
+
+func TestNewSyncerYAMLWithSyncerVersionChecksDisabled(t *testing.T) {
+	expectedYAML := `---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k-token
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+  annotations:
+    kubernetes.io/service-account.name: kcp-syncer-sync-target-name-34b23c4k
+type: kubernetes.io/service-account-token
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  verbs:
+  - "create"
+  - "list"
+  - "watch"
+  - "delete"
+- apiGroups:
+  - "apiextensions.k8s.io"
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - "get"
+  - "watch"
+  - "list"
+- apiGroups:
+  - ""
+  resources:
+  - resource1
+  - resource2
+  verbs:
+  - "*"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kcp-syncer-sync-target-name-34b23c4k
+subjects:
+- kind: ServiceAccount
+  name: kcp-syncer-sync-target-name-34b23c4k
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kcp-dns-sync-target-name-34b23c4k
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  - services
+  verbs:
+  - "create"
+  - "get"
+  - "list"
+  - "update"
+  - "delete"
+  - "watch"
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - "get"
+  - "list"
+  - "watch"
+- apiGroups:
+  - "apps"
+  resources:
+  - deployments
+  verbs:
+  - "create"
+  - "get"
+  - "list"
+  - "update"
+  - "delete"
+  - "watch"
+- apiGroups:
+  - "rbac.authorization.k8s.io"
+  resources:
+  - roles
+  - rolebindings
+  verbs:
+  - "create"
+  - "get"
+  - "list"
+  - "update"
+  - "delete"
+  - "watch"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kcp-dns-sync-target-name-34b23c4k
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kcp-dns-sync-target-name-34b23c4k
+subjects:
+  - kind: ServiceAccount
+    name: kcp-syncer-sync-target-name-34b23c4k
+    namespace: kcp-syncer-sync-target-name-34b23c4k
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kcp-syncer-sync-target-name-34b23c4k
+  namespace: kcp-syncer-sync-target-name-34b23c4k
+stringData:
+  kubeconfig: |
+    apiVersion: v1
+    kind: Config
+    clusters:
+    - name: default-cluster
+      cluster:
+        certificate-authority-data: ca-data
+        server: server-url
+    contexts:
+    - name: default-context
+      context:
+        cluster: default-cluster
+        namespace: kcp-namespace
+        user: default-user
+    current-context: default-context
+    users:
+    - name: default-user
+      user:
+        token: token
+---
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   name: kcp-syncer-sync-target-name-34b23c4k
   namespace: kcp-syncer-sync-target-name-34b23c4k
@@ -438,6 +667,7 @@ spec:
         - --burst=456
         - --feature-gates=myfeature=true
         - --dns-image=image
+        - --expectedKCPVersion=the-kcp-version
         env:
         - name: NAMESPACE
           valueFrom:
@@ -473,6 +703,7 @@ spec:
 		Burst:                       456,
 		APIImportPollIntervalString: "1m",
 		FeatureGatesString:          "myfeature=true",
+		ExpectedKCPVersion:          "the-kcp-version",
 	}, "kcp-syncer-sync-target-name-34b23c4k", []string{"resource1", "resource2"})
 	require.NoError(t, err)
 	require.Empty(t, cmp.Diff(expectedYAML, string(actualYAML)))

pkg/cliplugins/workload/plugin/syncer.yaml

@@ -191,6 +191,9 @@ spec:
         - --feature-gates={{ .FeatureGatesString }}
 {{- end}}
         - --dns-image={{.Image}}
+{{- if .ExpectedKCPVersion }}
+        - --expectedKCPVersion={{.ExpectedKCPVersion}}
+{{- end}}
         env:
         - name: NAMESPACE
           valueFrom:

pkg/syncer/syncer.go

@@ -42,6 +42,8 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
 
+	conditionsapi "github.com/kcp-dev/kcp/pkg/apis/third_party/conditions/apis/conditions/v1alpha1"
+	"github.com/kcp-dev/kcp/pkg/apis/third_party/conditions/util/conditions"
 	workloadv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/workload/v1alpha1"
 	kcpclient "github.com/kcp-dev/kcp/pkg/client/clientset/versioned"
 	kcpinformers "github.com/kcp-dev/kcp/pkg/client/informers/externalversions"
@@ -75,6 +77,7 @@ type SyncerConfig struct {
 	SyncTargetUID                 string
 	DownstreamNamespaceCleanDelay time.Duration
 	DNSImage                      string
+	ExpectedKCPVersion            string
 }
 
 func StartSyncer(ctx context.Context, cfg *SyncerConfig, numSyncerThreads int, importPollInterval time.Duration) error {
@@ -83,13 +86,23 @@ func StartSyncer(ctx context.Context, cfg *SyncerConfig, numSyncerThreads int, i
 	logger.V(2).Info("starting syncer")
 
 	kcpVersion := version.Get().GitVersion
-
 	kcpClusterClient, err := kcpclient.NewClusterForConfig(rest.AddUserAgent(rest.CopyConfig(cfg.UpstreamConfig), "kcp#syncer/"+kcpVersion))
 	if err != nil {
 		return err
 	}
 	kcpClient := kcpClusterClient.Cluster(cfg.SyncTargetWorkspace)
 
+	if cfg.ExpectedKCPVersion != "" && kcpVersion != cfg.ExpectedKCPVersion {
+		syncTarget, err := kcpClient.WorkloadV1alpha1().SyncTargets().Get(ctx, cfg.SyncTargetName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+		conditions.MarkFalse(syncTarget, workloadv1alpha1.SyncerReady, workloadv1alpha1.VersionMismatchReason, conditionsapi.ConditionSeverityError, "compiled version of the syncer (%s) does not match with the expected version set in the syncer deployment (%s)", kcpVersion, cfg.ExpectedKCPVersion)
+		if _, err := kcpClient.WorkloadV1alpha1().SyncTargets().Update(ctx, syncTarget, metav1.UpdateOptions{}); err != nil {
+			return err
+		}
+	}
+
 	dnsNamespace := os.Getenv("NAMESPACE")
 	if dnsNamespace == "" {
 		return errors.New("missing environment variable: NAMESPACE")

test/e2e/framework/syncer.go

@@ -45,6 +45,7 @@ import (
 	kubernetesclient "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/component-base/version"
 	"sigs.k8s.io/yaml"
 
 	apiresourcev1alpha1 "github.com/kcp-dev/kcp/pkg/apis/apiresource/v1alpha1"
@@ -480,6 +481,7 @@ func syncerConfigFromCluster(t *testing.T, downstreamConfig *rest.Config, namesp
 		SyncTargetUID:                 syncTargetUID,
 		DNSImage:                      dnsImage,
 		DownstreamNamespaceCleanDelay: 2 * time.Second,
+		ExpectedKCPVersion:            version.Get().GitVersion,
 	}
 }