add kubelet certificate mode in yurthub

cmd/yurthub/app/config/config.go

@@ -44,10 +44,9 @@ import (
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	ipUtils "github.com/openyurtio/openyurt/pkg/util/ip"
 	"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
 	"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
-	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
+	certificatemgr "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager"
 	"github.com/openyurtio/openyurt/pkg/yurthub/filter"
 	"github.com/openyurtio/openyurt/pkg/yurthub/filter/manager"
 	"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta"
@@ -178,10 +177,21 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
 		LeaderElection:            options.LeaderElection,
 	}
 
-	certMgr, err := createCertManager(options, us)
+	certMgr, err := certificatemgr.NewYurtHubCertManager(options, us)
 	if err != nil {
 		return nil, err
 	}
+	certMgr.Start()
+	err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) {
+		isReady := certMgr.Ready()
+		if isReady {
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("hub certificates preparation failed, %v", err)
+	}
 	cfg.CertManager = certMgr
 
 	if options.EnableDummyIf {
@@ -230,7 +240,7 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
 	return us, nil
 }
 
-// createSharedInformers create sharedInformers from the given proxyAddr.
+// createClientAndSharedInformers create kubeclient and sharedInformers from the given proxyAddr.
 func createClientAndSharedInformers(proxyAddr string, enableNodePool bool) (kubernetes.Interface, informers.SharedInformerFactory, yurtinformers.SharedInformerFactory, error) {
 	var kubeConfig *rest.Config
 	var yurtClient yurtclientset.Interface
@@ -341,45 +351,6 @@ func isServiceTopologyFilterEnabled(options *options.YurtHubOptions) bool {
 	return true
 }
 
-func createCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (certificate.YurtCertificateManager, error) {
-	// use dummy ip and bind ip as cert IP SANs
-	certIPs := ipUtils.RemoveDupIPs([]net.IP{
-		net.ParseIP(options.HubAgentDummyIfIP),
-		net.ParseIP(options.YurtHubHost),
-		net.ParseIP(options.YurtHubProxyHost),
-	})
-
-	cfg := &token.CertificateManagerConfiguration{
-		RootDir:                  options.RootDir,
-		NodeName:                 options.NodeName,
-		JoinToken:                options.JoinToken,
-		BootstrapFile:            options.BootstrapFile,
-		CaCertHashes:             options.CACertHashes,
-		YurtHubCertOrganizations: options.YurtHubCertOrganizations,
-		CertIPs:                  certIPs,
-		RemoteServers:            remoteServers,
-		Client:                   options.ClientForTest,
-	}
-	certManager, err := token.NewYurtHubCertManager(cfg)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create cert manager for yurthub, %v", err)
-	}
-
-	certManager.Start()
-	err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) {
-		isReady := certManager.Ready()
-		if isReady {
-			return true, nil
-		}
-		return false, nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("hub certificates preparation failed, %v", err)
-	}
-
-	return certManager, nil
-}
-
 func prepareServerServing(options *options.YurtHubOptions, certMgr certificate.YurtCertificateManager, cfg *YurtHubConfiguration) error {
 	if err := (&apiserveroptions.DeprecatedInsecureServingOptions{
 		BindAddress: net.ParseIP(options.YurtHubHost),

cmd/yurthub/app/config/config_test.go

@@ -20,12 +20,12 @@ import (
 	"testing"
 
 	"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
-	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata"
+	"github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata"
 )
 
 func TestComplete(t *testing.T) {
 	options := options.NewYurtHubOptions()
-	client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/token/testdata")
+	client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/testdata")
 	if err != nil {
 		t.Errorf("failed to create cert fake client, %v", err)
 		return

cmd/yurthub/app/options/options.go

@@ -62,6 +62,7 @@ type YurtHubOptions struct {
 	HeartbeatIntervalSeconds  int
 	MaxRequestInFlight        int
 	JoinToken                 string
+	BootstrapMode             string
 	BootstrapFile             string
 	RootDir                   string
 	Version                   bool
@@ -105,6 +106,7 @@ func NewYurtHubOptions() *YurtHubOptions {
 		HeartbeatTimeoutSeconds:   2,
 		HeartbeatIntervalSeconds:  10,
 		MaxRequestInFlight:        250,
+		BootstrapMode:             "token",
 		RootDir:                   filepath.Join("/var/lib/", projectinfo.GetHubName()),
 		EnableProfiling:           true,
 		EnableDummyIf:             true,
@@ -189,6 +191,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&o.MaxRequestInFlight, "max-requests-in-flight", o.MaxRequestInFlight, "the maximum number of parallel requests.")
 	fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent.")
 	fs.MarkDeprecated("join-token", "It is planned to be removed from OpenYurt in the version v1.5. Please use --bootstrap-file to bootstrap hub agent.")
+	fs.StringVar(&o.BootstrapMode, "bootstrap-mode", o.BootstrapMode, "the mode for bootstrapping hub agent(token, kubeletcertificate).")
 	fs.StringVar(&o.BootstrapFile, "bootstrap-file", o.BootstrapFile, "the bootstrap file for bootstrapping hub agent.")
 	fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).")
 	fs.BoolVar(&o.Version, "version", o.Version, "print the version information.")

cmd/yurthub/app/options/options_test.go

@@ -49,6 +49,7 @@ func TestNewYurtHubOptions(t *testing.T) {
 		HeartbeatTimeoutSeconds:   2,
 		HeartbeatIntervalSeconds:  10,
 		MaxRequestInFlight:        250,
+		BootstrapMode:             "token",
 		RootDir:                   filepath.Join("/var/lib/", projectinfo.GetHubName()),
 		EnableProfiling:           true,
 		EnableDummyIf:             true,

pkg/yurthub/certificate/interfaces.go

@@ -22,14 +22,25 @@ import (
 
 // YurtCertificateManager is responsible for managing node certificate for yurthub
 type YurtCertificateManager interface {
-	Start()
-	Stop()
+	YurtClientCertificateManager
+	YurtServerCertificateManager
 	// Ready should be called after yurt certificate manager started by Start.
 	Ready() bool
+}
+
+// YurtClientCertificateManager is responsible for managing node client certificates for yurthub
+type YurtClientCertificateManager interface {
+	Start()
+	Stop()
 	UpdateBootstrapConf(joinToken string) error
 	GetHubConfFile() string
 	GetCaFile() string
 	GetAPIServerClientCert() *tls.Certificate
+}
+
+type YurtServerCertificateManager interface {
+	Start()
+	Stop()
 	GetHubServerCert() *tls.Certificate
 	GetHubServerCertFile() string
 }

pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeletcertificate
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
+	"github.com/openyurtio/openyurt/pkg/yurthub/util"
+)
+
+var (
+	KubeConfNotExistErr   = errors.New("/etc/kubernetes/kubelet.conf file doesn't exist")
+	KubeletCANotExistErr  = errors.New("/etc/kubernetes/pki/ca.crt file doesn't exist")
+	KubeletPemNotExistErr = errors.New("/var/lib/kubelet/pki/kubelet-current.pem file doesn't exist")
+)
+
+type kubeletCertManager struct {
+	kubeConfFile   string
+	kubeletCAFile  string
+	kubeletPemFile string
+	cert           *tls.Certificate
+}
+
+func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) {
+	if exist, _ := util.FileExists(kubeConfFile); !exist {
+		return nil, KubeConfNotExistErr
+	}
+
+	if exist, _ := util.FileExists(kubeletCAFile); !exist {
+		return nil, KubeletCANotExistErr
+	}
+
+	if exist, _ := util.FileExists(kubeletPemFile); !exist {
+		return nil, KubeletPemNotExistErr
+	}
+
+	cert, err := loadFile(kubeletPemFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return &kubeletCertManager{
+		kubeConfFile:   kubeConfFile,
+		kubeletCAFile:  kubeletCAFile,
+		kubeletPemFile: kubeletPemFile,
+		cert:           cert,
+	}, nil
+}
+
+func (kcm *kubeletCertManager) Start() {
+	// do nothing
+}
+
+func (kcm *kubeletCertManager) Stop() {
+	// do nothing
+}
+
+func (kcm *kubeletCertManager) UpdateBootstrapConf(_ string) error {
+	return nil
+}
+
+func (kcm *kubeletCertManager) GetHubConfFile() string {
+	return kcm.kubeConfFile
+}
+
+func (kcm *kubeletCertManager) GetCaFile() string {
+	return kcm.kubeletCAFile
+}
+
+func (kcm *kubeletCertManager) GetAPIServerClientCert() *tls.Certificate {
+	if kcm.cert != nil && kcm.cert.Leaf != nil && !time.Now().After(kcm.cert.Leaf.NotAfter) {
+		return kcm.cert
+	}
+
+	klog.Warningf("current certificate: %s is expired, reload it", kcm.kubeletPemFile)
+	cert, err := loadFile(kcm.kubeletPemFile)
+	if err != nil {
+		klog.Errorf("failed to load client certificate(%s), %v", kcm.kubeletPemFile, err)
+		return nil
+	}
+	kcm.cert = cert
+	return kcm.cert
+}
+
+func loadFile(pairFile string) (*tls.Certificate, error) {
+	// LoadX509KeyPair knows how to parse combined cert and private key from
+	// the same file.
+	cert, err := tls.LoadX509KeyPair(pairFile, pairFile)
+	if err != nil {
+		return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err)
+	}
+	certs, err := x509.ParseCertificates(cert.Certificate[0])
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse certificate data: %v", err)
+	}
+	cert.Leaf = certs[0]
+	return &cert, nil
+}

pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2023 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeletcertificate
+
+import "testing"
+
+func TestNewKubeletCertManager(t *testing.T) {
+	testcases := map[string]struct {
+		kubeConfFile   string
+		kubeletCAFile  string
+		kubeletPemFile string
+		err            error
+	}{
+		"kubelet.conf doesn't exist": {
+			kubeConfFile: "invalid file",
+			err:          KubeConfNotExistErr,
+		},
+		"ca.crt file doesn't exist": {
+			kubeConfFile:  "../testdata/kubelet.conf",
+			kubeletCAFile: "invalid file",
+			err:           KubeletCANotExistErr,
+		},
+		"kubelet.pem doesn't exist": {
+			kubeConfFile:   "../testdata/kubelet.conf",
+			kubeletCAFile:  "../testdata/ca.crt",
+			kubeletPemFile: "invalid file",
+			err:            KubeletPemNotExistErr,
+		},
+		"normal kubelet cert manager": {
+			kubeConfFile:   "../testdata/kubelet.conf",
+			kubeletCAFile:  "../testdata/ca.crt",
+			kubeletPemFile: "../testdata/kubelet.pem",
+			err:            nil,
+		},
+	}
+
+	for k, tc := range testcases {
+		t.Run(k, func(t *testing.T) {
+			_, err := NewKubeletCertManager(tc.kubeConfFile, tc.kubeletCAFile, tc.kubeletPemFile)
+			if err != tc.err {
+				t.Errorf("expect error is %v, but got %v", tc.err, err)
+			}
+		})
+	}
+}