Apply OpenYurt to support K8s v1.28

.github/workflows/ci.yaml

@@ -9,7 +9,8 @@ on:
   workflow_dispatch: {}
 
 env:
-  GO_VERSION: 1.18
+  GO_VERSION: '1.20'
+  GOLANGCI_VERSION: 'v1.55.2'
 
   # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run
   # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether
@@ -18,7 +19,6 @@ env:
   AWS_USR: ${{ secrets.AWS_USR }}
 
 jobs:
-
   verify:
     runs-on: ubuntu-22.04
     steps:
@@ -42,11 +42,12 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Lint golang code
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6.0.1
         with:
-          version: v1.54
-          args: -v
+          version: ${{ env.GOLANGCI_VERSION }}
+          args: --verbose
           skip-cache: true
+          mode: readonly
 
   markdownlint-misspell-shellcheck:
     runs-on: ubuntu-22.04
@@ -120,8 +121,8 @@ jobs:
 #          restore-keys: ${{ runner.os }}-go-
       - name: Install Required Commands
         run: |
-          go get sigs.k8s.io/kind@v0.12.0
-          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
+          go install sigs.k8s.io/kind@v0.22.0
+          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.28.7/bin/linux/amd64/kubectl && sudo install kubectl /usr/local/bin/kubectl
       - name: Build Images
         run: make docker-build
       - name: Local Up Openyurt Cluster With Kind

.gitignore

@@ -4,6 +4,7 @@ bin
 .DS_Store
 gopath
 dockerbuild
+hack/cni
 
 vendor
 .vscode

Makefile

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-KUBERNETESVERSION ?=v1.22
-GOLANGCILINT_VERSION ?= v1.54
+KUBERNETESVERSION ?=v1.28
+GOLANGCILINT_VERSION ?= v1.55.2
 GLOBAL_GOLANGCILINT := $(shell which golangci-lint)
 GOBIN := $(shell go env GOPATH)/bin
 GOBIN_GOLANGCILINT := $(shell which $(GOBIN)/golangci-lint)
@@ -59,7 +59,7 @@ KUSTOMIZE_VERSION ?= v4.5.7
 ## Tool Binaries
 KUSTOMIZE ?= $(LOCALBIN)/kustomize
 
-KUBECTL_VERSION ?= v1.22.3
+KUBECTL_VERSION ?= v1.28.7
 KUBECTL ?= $(LOCALBIN)/kubectl
 
 YQ_VERSION := 4.13.2
@@ -209,7 +209,7 @@ $(KUBECTL): $(LOCALBIN)
 		echo "$(LOCALBIN)/kubectl version is not expected $(KUBECTL_VERSION). Removing it before installing."; \
 		rm -rf $(LOCALBIN)/kubectl; \
 	fi
-	test -s $(LOCALBIN)/kubectl || curl https://storage.googleapis.com/kubernetes-release/release/v1.22.3/bin/$(shell go env GOOS)/$(shell go env GOARCH)/kubectl -o $(KUBECTL)
+	test -s $(LOCALBIN)/kubectl || curl https://storage.googleapis.com/kubernetes-release/release/$(KUBECTL_VERSION)/bin/$(shell go env GOOS)/$(shell go env GOARCH)/kubectl -o $(KUBECTL)
 	chmod +x $(KUBECTL)
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"

charts/yurt-manager/crds/network.openyurt.io_poolservices.yaml

@@ -71,13 +71,14 @@ spec:
                   description: "Condition contains details for one aspect of the current
                     state of this API Resource. --- This struct is intended for direct
                     use as an array at the field path .status.conditions.  For example,
-                    type FooStatus struct{     // Represents the observations of a
-                    foo's current state.     // Known .status.conditions.type are:
-                    \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type
-                    \    // +patchStrategy=merge     // +listType=map     // +listMapKey=type
-                    \    Conditions []metav1.Condition `json:\"conditions,omitempty\"
-                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                    \n     // other fields }"
+                    \n \ttype FooStatus struct{ \t    // Represents the observations
+                    of a foo's current state. \t    // Known .status.conditions.type
+                    are: \"Available\", \"Progressing\", and \"Degraded\" \t    //
+                    +patchMergeKey=type \t    // +patchStrategy=merge \t    // +listType=map
+                    \t    // +listMapKey=type \t    Conditions []metav1.Condition
+                    `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\"
+                    protobuf:\"bytes,1,rep,name=conditions\"` \n \t    // other fields
+                    \t}"
                   properties:
                     lastTransitionTime:
                       description: lastTransitionTime is the last time the condition

cmd/yurt-iot-dock/app/core.go

@@ -19,6 +19,7 @@ package app
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
 	"syscall"
@@ -34,6 +35,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/openyurtio/openyurt/cmd/yurt-iot-dock/app/options"
 	"github.com/openyurtio/openyurt/pkg/apis"
@@ -81,13 +83,17 @@ func Run(opts *options.YurtIoTDockOptions, stopCh <-chan struct{}) {
 	ctrl.SetLogger(klogr.New())
 	cfg := ctrl.GetConfigOrDie()
 
+	metricsServerOpts := metricsserver.Options{
+		BindAddress:   opts.MetricsAddr,
+		ExtraHandlers: make(map[string]http.Handler, 0),
+	}
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:                 scheme,
-		MetricsBindAddress:     opts.MetricsAddr,
+		Metrics:                metricsServerOpts,
 		HealthProbeBindAddress: opts.ProbeAddr,
 		LeaderElection:         opts.EnableLeaderElection,
 		LeaderElectionID:       "yurt-iot-dock",
-		Namespace:              opts.Namespace,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")

cmd/yurt-iot-dock/yurt-iot-dock.go

@@ -18,8 +18,6 @@ package main
 
 import (
 	"flag"
-	"math/rand"
-	"time"
 
 	"k8s.io/apimachinery/pkg/util/wait"
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
@@ -31,7 +29,6 @@ import (
 )
 
 func main() {
-	rand.Seed(time.Now().UnixNano())
 	klog.InitFlags(nil)
 	defer klog.Flush()
 

cmd/yurt-manager/app/manager.go

@@ -18,6 +18,7 @@ package app
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 
 	"github.com/spf13/cobra"
@@ -34,6 +35,8 @@ import (
 	"k8s.io/klog/v2/klogr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+	runtimewebhook "sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	"github.com/openyurtio/openyurt/cmd/yurt-manager/app/config"
 	"github.com/openyurtio/openyurt/cmd/yurt-manager/app/options"
@@ -160,19 +163,28 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 	}
 	setRestConfig(cfg, c)
 
+	metricsServerOpts := metricsserver.Options{
+		BindAddress:   c.ComponentConfig.Generic.MetricsAddr,
+		ExtraHandlers: make(map[string]http.Handler, 0),
+	}
+	for path, handler := range profile.GetPprofHandlers() {
+		metricsServerOpts.ExtraHandlers[path] = handler
+	}
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:                     scheme,
-		MetricsBindAddress:         c.ComponentConfig.Generic.MetricsAddr,
+		Metrics:                    metricsServerOpts,
 		HealthProbeBindAddress:     c.ComponentConfig.Generic.HealthProbeAddr,
 		LeaderElection:             c.ComponentConfig.Generic.LeaderElection.LeaderElect,
 		LeaderElectionID:           c.ComponentConfig.Generic.LeaderElection.ResourceName,
 		LeaderElectionNamespace:    c.ComponentConfig.Generic.LeaderElection.ResourceNamespace,
 		LeaderElectionResourceLock: c.ComponentConfig.Generic.LeaderElection.ResourceLock,
-		Port:                       util.GetWebHookPort(),
-		Namespace:                  "",
-		Logger:                     setupLog,
-		CertDir:                    util.GetCertDir(),
-		Host:                       "0.0.0.0",
+		WebhookServer: runtimewebhook.NewServer(runtimewebhook.Options{
+			Host:    "0.0.0.0",
+			Port:    util.GetWebHookPort(),
+			CertDir: util.GetCertDir(),
+		}),
+		Logger: setupLog,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
@@ -216,13 +228,6 @@ func Run(c *config.CompletedConfig, stopCh <-chan struct{}) error {
 		os.Exit(1)
 	}
 
-	for path, handler := range profile.GetPprofHandlers() {
-		if err := mgr.AddMetricsExtraHandler(path, handler); err != nil {
-			setupLog.Error(err, "unable to add pprof handler")
-			os.Exit(1)
-		}
-	}
-
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")

cmd/yurt-tunnel-agent/app/start.go

@@ -17,6 +17,7 @@ limitations under the License.
 package app
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -111,14 +112,14 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 	agentCertMgr.Start()
 
 	// 2.1. waiting for the certificate is generated
-	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
+	_ = wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) {
 		if agentCertMgr.Current() != nil {
 			return true, nil
 		}
 		klog.Infof("certificate %s not signed, waiting...",
 			projectinfo.GetAgentName())
 		return false, nil
-	}, stopCh)
+	})
 	klog.Infof("certificate %s ok", projectinfo.GetAgentName())
 
 	// 3. generate a TLS configuration for securing the connection to server

cmd/yurt-tunnel-server/app/config/config.go

@@ -76,5 +76,5 @@ func (c *Config) Complete() *CompletedConfig {
 }
 
 func (c *Config) IsIPv6() bool {
-	return c.IPFamily == iptables.ProtocolIpv6
+	return c.IPFamily == iptables.ProtocolIPv6
 }

cmd/yurt-tunnel-server/app/options/options.go

@@ -140,9 +140,9 @@ func (o *ServerOptions) Config() (*config.Config, error) {
 	}
 
 	if utilnet.IsIPv6String(o.BindAddr) {
-		cfg.IPFamily = iptables.ProtocolIpv6
+		cfg.IPFamily = iptables.ProtocolIPv6
 	} else {
-		cfg.IPFamily = iptables.ProtocolIpv4
+		cfg.IPFamily = iptables.ProtocolIPv4
 	}
 	cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort)
 	cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort)

cmd/yurt-tunnel-server/app/start.go

@@ -167,14 +167,14 @@ func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
 	cfg.SharedInformerFactory.Start(stopCh)
 
 	// 5. waiting for the certificate is generated
-	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
+	_ = wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) {
 		// keep polling until the certificate is signed
 		if serverCertMgr.Current() != nil && tunnelProxyCertMgr.Current() != nil {
 			return true, nil
 		}
 		klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetServerName())
 		return false, nil
-	}, stopCh)
+	})
 
 	// 6. generate the TLS configuration based on the latest certificate
 	tlsCfg, err := certmanager.GenTLSConfigUseCurrentCertAndCertPool(serverCertMgr.Current, cfg.RootCert, "server")
@@ -219,7 +219,7 @@ func getTunnelServerIPsAndDNSNamesBeforeInformerSynced(clientset kubernetes.Inte
 	)
 
 	// the ips and dnsNames should be acquired through api-server at the first time, because the informer factory has not started yet.
-	werr := wait.PollUntil(5*time.Second, func() (bool, error) {
+	werr := wait.PollUntilContextCancel(context.Background(), 5*time.Second, true, func(ctx context.Context) (bool, error) {
 		dnsNames, ips, err = serveraddr.GetYurttunelServerDNSandIP(clientset)
 		if err != nil {
 			klog.Errorf("failed to get yurt tunnel server dns and ip, %v", err)
@@ -243,7 +243,7 @@ func getTunnelServerIPsAndDNSNamesBeforeInformerSynced(clientset kubernetes.Inte
 		}
 
 		return true, nil
-	}, stopCh)
+	})
 	if werr != nil {
 		return nil, nil, werr
 	}

cmd/yurthub/app/config/config.go

@@ -17,6 +17,7 @@ limitations under the License.
 package config
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -178,7 +179,7 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		return nil, err
 	}
 	certMgr.Start()
-	err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) {
+	err = wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 4*time.Minute, true, func(ctx context.Context) (bool, error) {
 		isReady := certMgr.Ready()
 		if isReady {
 			return true, nil

cmd/yurthub/app/start.go

@@ -29,6 +29,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/klog/v2"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/config"
@@ -84,6 +85,7 @@ func NewCmdStartYurtHub(ctx context.Context) *cobra.Command {
 		},
 	}
 
+	globalflag.AddGlobalFlags(cmd.Flags(), cmd.Name())
 	yurtHubOptions.AddFlags(cmd.Flags())
 	return cmd
 }
@@ -350,7 +352,7 @@ func coordinatorRun(ctx context.Context,
 }
 
 func yurtCoordinatorTransportMgrGetter(coordinatorCertMgr *coordinatorcertmgr.CertManager, stopCh <-chan struct{}) (transport.Interface, error) {
-	err := wait.PollImmediate(5*time.Second, 4*time.Minute, func() (done bool, err error) {
+	err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 4*time.Minute, true, func(ctx context.Context) (done bool, err error) {
 		klog.Info("waiting for preparing certificates for coordinator client and node lease proxy client")
 		if coordinatorCertMgr.GetAPIServerClientCert() == nil {
 			return false, nil