From 4efa1bf02c689f2746a88547dfdff32a04b3a561 Mon Sep 17 00:00:00 2001 From: rambohe Date: Mon, 24 Jul 2023 11:01:19 +0800 Subject: [PATCH] add kubelet certificate mode in yurthub (#1625) --- cmd/yurthub/app/config/config.go | 57 ++----- cmd/yurthub/app/config/config_test.go | 4 +- cmd/yurthub/app/options/options.go | 3 + cmd/yurthub/app/options/options_test.go | 1 + pkg/yurthub/certificate/interfaces.go | 15 +- .../kubeletcertificate/kubelet_certificate.go | 119 ++++++++++++++ .../kubelet_certificate_test.go | 59 +++++++ pkg/yurthub/certificate/manager/manager.go | 137 ++++++++++++++++ .../certificate/manager/manager_test.go | 136 +++++++++++++++ pkg/yurthub/certificate/server/server.go | 102 ++++++++++++ pkg/yurthub/certificate/server/server_test.go | 55 +++++++ .../certificate/{token => }/testdata/ca.crt | 0 .../certificate/{token => }/testdata/ca.key | 0 .../{token => }/testdata/fake_client.go | 0 pkg/yurthub/certificate/testdata/kubelet.conf | 14 ++ pkg/yurthub/certificate/testdata/kubelet.pem | 22 +++ pkg/yurthub/certificate/token/token.go | 132 +++------------ pkg/yurthub/certificate/token/token_test.go | 155 +++--------------- pkg/yurthub/kubernetes/rest/config_test.go | 18 +- pkg/yurthub/server/certificate_test.go | 19 +-- 20 files changed, 733 insertions(+), 315 deletions(-) create mode 100644 pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go create mode 100644 pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go create mode 100644 pkg/yurthub/certificate/manager/manager.go create mode 100644 pkg/yurthub/certificate/manager/manager_test.go create mode 100644 pkg/yurthub/certificate/server/server.go create mode 100644 pkg/yurthub/certificate/server/server_test.go rename pkg/yurthub/certificate/{token => }/testdata/ca.crt (100%) rename pkg/yurthub/certificate/{token => }/testdata/ca.key (100%) rename pkg/yurthub/certificate/{token => }/testdata/fake_client.go (100%) create mode 100644 pkg/yurthub/certificate/testdata/kubelet.conf create mode 100644 pkg/yurthub/certificate/testdata/kubelet.pem diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go index 85a64ab435a..06529a69fc5 100644 --- a/cmd/yurthub/app/config/config.go +++ b/cmd/yurthub/app/config/config.go @@ -44,10 +44,9 @@ import ( "github.com/openyurtio/openyurt/cmd/yurthub/app/options" "github.com/openyurtio/openyurt/pkg/projectinfo" - ipUtils "github.com/openyurtio/openyurt/pkg/util/ip" "github.com/openyurtio/openyurt/pkg/yurthub/cachemanager" "github.com/openyurtio/openyurt/pkg/yurthub/certificate" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + certificatemgr "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" "github.com/openyurtio/openyurt/pkg/yurthub/filter" "github.com/openyurtio/openyurt/pkg/yurthub/filter/manager" "github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta" @@ -178,10 +177,21 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) { LeaderElection: options.LeaderElection, } - certMgr, err := createCertManager(options, us) + certMgr, err := certificatemgr.NewYurtHubCertManager(options, us) if err != nil { return nil, err } + certMgr.Start() + err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) { + isReady := certMgr.Ready() + if isReady { + return true, nil + } + return false, nil + }) + if err != nil { + return nil, fmt.Errorf("hub certificates preparation failed, %v", err) + } cfg.CertManager = certMgr if options.EnableDummyIf { @@ -230,7 +240,7 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) { return us, nil } -// createSharedInformers create sharedInformers from the given proxyAddr. +// createClientAndSharedInformers create kubeclient and sharedInformers from the given proxyAddr. func createClientAndSharedInformers(proxyAddr string, enableNodePool bool) (kubernetes.Interface, informers.SharedInformerFactory, yurtinformers.SharedInformerFactory, error) { var kubeConfig *rest.Config var yurtClient yurtclientset.Interface @@ -341,45 +351,6 @@ func isServiceTopologyFilterEnabled(options *options.YurtHubOptions) bool { return true } -func createCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (certificate.YurtCertificateManager, error) { - // use dummy ip and bind ip as cert IP SANs - certIPs := ipUtils.RemoveDupIPs([]net.IP{ - net.ParseIP(options.HubAgentDummyIfIP), - net.ParseIP(options.YurtHubHost), - net.ParseIP(options.YurtHubProxyHost), - }) - - cfg := &token.CertificateManagerConfiguration{ - RootDir: options.RootDir, - NodeName: options.NodeName, - JoinToken: options.JoinToken, - BootstrapFile: options.BootstrapFile, - CaCertHashes: options.CACertHashes, - YurtHubCertOrganizations: options.YurtHubCertOrganizations, - CertIPs: certIPs, - RemoteServers: remoteServers, - Client: options.ClientForTest, - } - certManager, err := token.NewYurtHubCertManager(cfg) - if err != nil { - return nil, fmt.Errorf("failed to create cert manager for yurthub, %v", err) - } - - certManager.Start() - err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) { - isReady := certManager.Ready() - if isReady { - return true, nil - } - return false, nil - }) - if err != nil { - return nil, fmt.Errorf("hub certificates preparation failed, %v", err) - } - - return certManager, nil -} - func prepareServerServing(options *options.YurtHubOptions, certMgr certificate.YurtCertificateManager, cfg *YurtHubConfiguration) error { if err := (&apiserveroptions.DeprecatedInsecureServingOptions{ BindAddress: net.ParseIP(options.YurtHubHost), diff --git a/cmd/yurthub/app/config/config_test.go b/cmd/yurthub/app/config/config_test.go index cd657071e51..3fd46d2108e 100644 --- a/cmd/yurthub/app/config/config_test.go +++ b/cmd/yurthub/app/config/config_test.go @@ -20,12 +20,12 @@ import ( "testing" "github.com/openyurtio/openyurt/cmd/yurthub/app/options" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" ) func TestComplete(t *testing.T) { options := options.NewYurtHubOptions() - client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/token/testdata") + client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go index ccabdbdd097..a6f0deadbaf 100644 --- a/cmd/yurthub/app/options/options.go +++ b/cmd/yurthub/app/options/options.go @@ -62,6 +62,7 @@ type YurtHubOptions struct { HeartbeatIntervalSeconds int MaxRequestInFlight int JoinToken string + BootstrapMode string BootstrapFile string RootDir string Version bool @@ -105,6 +106,7 @@ func NewYurtHubOptions() *YurtHubOptions { HeartbeatTimeoutSeconds: 2, HeartbeatIntervalSeconds: 10, MaxRequestInFlight: 250, + BootstrapMode: "token", RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()), EnableProfiling: true, EnableDummyIf: true, @@ -189,6 +191,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.MaxRequestInFlight, "max-requests-in-flight", o.MaxRequestInFlight, "the maximum number of parallel requests.") fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent.") fs.MarkDeprecated("join-token", "It is planned to be removed from OpenYurt in the version v1.5. Please use --bootstrap-file to bootstrap hub agent.") + fs.StringVar(&o.BootstrapMode, "bootstrap-mode", o.BootstrapMode, "the mode for bootstrapping hub agent(token, kubeletcertificate).") fs.StringVar(&o.BootstrapFile, "bootstrap-file", o.BootstrapFile, "the bootstrap file for bootstrapping hub agent.") fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).") fs.BoolVar(&o.Version, "version", o.Version, "print the version information.") diff --git a/cmd/yurthub/app/options/options_test.go b/cmd/yurthub/app/options/options_test.go index d9d2379b600..a4370df98cc 100644 --- a/cmd/yurthub/app/options/options_test.go +++ b/cmd/yurthub/app/options/options_test.go @@ -49,6 +49,7 @@ func TestNewYurtHubOptions(t *testing.T) { HeartbeatTimeoutSeconds: 2, HeartbeatIntervalSeconds: 10, MaxRequestInFlight: 250, + BootstrapMode: "token", RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()), EnableProfiling: true, EnableDummyIf: true, diff --git a/pkg/yurthub/certificate/interfaces.go b/pkg/yurthub/certificate/interfaces.go index 6ac9d8a9bae..ecbaedc8220 100644 --- a/pkg/yurthub/certificate/interfaces.go +++ b/pkg/yurthub/certificate/interfaces.go @@ -22,14 +22,25 @@ import ( // YurtCertificateManager is responsible for managing node certificate for yurthub type YurtCertificateManager interface { - Start() - Stop() + YurtClientCertificateManager + YurtServerCertificateManager // Ready should be called after yurt certificate manager started by Start. Ready() bool +} + +// YurtClientCertificateManager is responsible for managing node client certificates for yurthub +type YurtClientCertificateManager interface { + Start() + Stop() UpdateBootstrapConf(joinToken string) error GetHubConfFile() string GetCaFile() string GetAPIServerClientCert() *tls.Certificate +} + +type YurtServerCertificateManager interface { + Start() + Stop() GetHubServerCert() *tls.Certificate GetHubServerCertFile() string } diff --git a/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go new file mode 100644 index 00000000000..00d1e7e911d --- /dev/null +++ b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go @@ -0,0 +1,119 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletcertificate + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "time" + + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/yurthub/certificate" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +var ( + KubeConfNotExistErr = errors.New("/etc/kubernetes/kubelet.conf file doesn't exist") + KubeletCANotExistErr = errors.New("/etc/kubernetes/pki/ca.crt file doesn't exist") + KubeletPemNotExistErr = errors.New("/var/lib/kubelet/pki/kubelet-current.pem file doesn't exist") +) + +type kubeletCertManager struct { + kubeConfFile string + kubeletCAFile string + kubeletPemFile string + cert *tls.Certificate +} + +func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) { + if exist, _ := util.FileExists(kubeConfFile); !exist { + return nil, KubeConfNotExistErr + } + + if exist, _ := util.FileExists(kubeletCAFile); !exist { + return nil, KubeletCANotExistErr + } + + if exist, _ := util.FileExists(kubeletPemFile); !exist { + return nil, KubeletPemNotExistErr + } + + cert, err := loadFile(kubeletPemFile) + if err != nil { + return nil, err + } + + return &kubeletCertManager{ + kubeConfFile: kubeConfFile, + kubeletCAFile: kubeletCAFile, + kubeletPemFile: kubeletPemFile, + cert: cert, + }, nil +} + +func (kcm *kubeletCertManager) Start() { + // do nothing +} + +func (kcm *kubeletCertManager) Stop() { + // do nothing +} + +func (kcm *kubeletCertManager) UpdateBootstrapConf(_ string) error { + return nil +} + +func (kcm *kubeletCertManager) GetHubConfFile() string { + return kcm.kubeConfFile +} + +func (kcm *kubeletCertManager) GetCaFile() string { + return kcm.kubeletCAFile +} + +func (kcm *kubeletCertManager) GetAPIServerClientCert() *tls.Certificate { + if kcm.cert != nil && kcm.cert.Leaf != nil && !time.Now().After(kcm.cert.Leaf.NotAfter) { + return kcm.cert + } + + klog.Warningf("current certificate: %s is expired, reload it", kcm.kubeletPemFile) + cert, err := loadFile(kcm.kubeletPemFile) + if err != nil { + klog.Errorf("failed to load client certificate(%s), %v", kcm.kubeletPemFile, err) + return nil + } + kcm.cert = cert + return kcm.cert +} + +func loadFile(pairFile string) (*tls.Certificate, error) { + // LoadX509KeyPair knows how to parse combined cert and private key from + // the same file. + cert, err := tls.LoadX509KeyPair(pairFile, pairFile) + if err != nil { + return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err) + } + certs, err := x509.ParseCertificates(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("unable to parse certificate data: %v", err) + } + cert.Leaf = certs[0] + return &cert, nil +} diff --git a/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go new file mode 100644 index 00000000000..c9e46a72506 --- /dev/null +++ b/pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeletcertificate + +import "testing" + +func TestNewKubeletCertManager(t *testing.T) { + testcases := map[string]struct { + kubeConfFile string + kubeletCAFile string + kubeletPemFile string + err error + }{ + "kubelet.conf doesn't exist": { + kubeConfFile: "invalid file", + err: KubeConfNotExistErr, + }, + "ca.crt file doesn't exist": { + kubeConfFile: "../testdata/kubelet.conf", + kubeletCAFile: "invalid file", + err: KubeletCANotExistErr, + }, + "kubelet.pem doesn't exist": { + kubeConfFile: "../testdata/kubelet.conf", + kubeletCAFile: "../testdata/ca.crt", + kubeletPemFile: "invalid file", + err: KubeletPemNotExistErr, + }, + "normal kubelet cert manager": { + kubeConfFile: "../testdata/kubelet.conf", + kubeletCAFile: "../testdata/ca.crt", + kubeletPemFile: "../testdata/kubelet.pem", + err: nil, + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + _, err := NewKubeletCertManager(tc.kubeConfFile, tc.kubeletCAFile, tc.kubeletPemFile) + if err != tc.err { + t.Errorf("expect error is %v, but got %v", tc.err, err) + } + }) + } +} diff --git a/pkg/yurthub/certificate/manager/manager.go b/pkg/yurthub/certificate/manager/manager.go new file mode 100644 index 00000000000..9d1f8cfff40 --- /dev/null +++ b/pkg/yurthub/certificate/manager/manager.go @@ -0,0 +1,137 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "errors" + "net" + "net/url" + "path/filepath" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/projectinfo" + ipUtils "github.com/openyurtio/openyurt/pkg/util/ip" + hubCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/kubeletcertificate" + hubServerCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate/server" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" + "github.com/openyurtio/openyurt/pkg/yurthub/util" +) + +const ( + KubeConfFile = "/etc/kubernetes/kubelet.conf" + KubeletCAFile = "/etc/kubernetes/pki/ca.crt" + KubeletPemFile = "/var/lib/kubelet/pki/current-kubelet.pem" +) + +var ( + serverCertNotReadyError = errors.New("hub server certificate") + apiServerClientCertNotReadyError = errors.New("APIServer client certificate") + caCertIsNotReadyError = errors.New("ca.crt file") + + DefaultRootDir = "/var/lib" +) + +type yurtHubCertManager struct { + hubCert.YurtClientCertificateManager + hubCert.YurtServerCertificateManager +} + +// NewYurtHubCertManager new a YurtCertificateManager instance +func NewYurtHubCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (hubCert.YurtCertificateManager, error) { + var clientCertManager hubCert.YurtClientCertificateManager + var err error + + workDir := filepath.Join(options.RootDir, projectinfo.GetHubName()) + if len(options.RootDir) == 0 { + workDir = filepath.Join(DefaultRootDir, projectinfo.GetHubName()) + } + + if options.BootstrapMode == "kubeletcertificate" { + clientCertManager, err = kubeletcertificate.NewKubeletCertManager(KubeConfFile, KubeletCAFile, KubeletPemFile) + if err != nil { + return nil, err + } + } else { + cfg := &token.ClientCertificateManagerConfiguration{ + WorkDir: workDir, + NodeName: options.NodeName, + JoinToken: options.JoinToken, + BootstrapFile: options.BootstrapFile, + CaCertHashes: options.CACertHashes, + YurtHubCertOrganizations: options.YurtHubCertOrganizations, + RemoteServers: remoteServers, + Client: options.ClientForTest, + } + clientCertManager, err = token.NewYurtHubClientCertManager(cfg) + if err != nil { + return nil, err + } + } + + // use dummy ip and bind ip as cert IP SANs + certIPs := ipUtils.RemoveDupIPs([]net.IP{ + net.ParseIP(options.HubAgentDummyIfIP), + net.ParseIP(options.YurtHubHost), + net.ParseIP(options.YurtHubProxyHost), + }) + serverCertManager, err := hubServerCert.NewHubServerCertificateManager(options.ClientForTest, clientCertManager, options.NodeName, filepath.Join(workDir, "pki"), certIPs) + if err != nil { + return nil, err + } + + hubCertManager := &yurtHubCertManager{ + YurtClientCertificateManager: clientCertManager, + YurtServerCertificateManager: serverCertManager, + } + + return hubCertManager, nil +} + +func (hcm *yurtHubCertManager) Start() { + hcm.YurtClientCertificateManager.Start() + hcm.YurtServerCertificateManager.Start() +} + +func (hcm *yurtHubCertManager) Stop() { + hcm.YurtClientCertificateManager.Stop() + hcm.YurtServerCertificateManager.Stop() +} + +func (hcm *yurtHubCertManager) Ready() bool { + var errs []error + if hcm.GetAPIServerClientCert() == nil { + errs = append(errs, apiServerClientCertNotReadyError) + } + + if exist, _ := util.FileExists(hcm.YurtClientCertificateManager.GetCaFile()); !exist { + errs = append(errs, caCertIsNotReadyError) + } + + if hcm.GetHubServerCert() == nil { + errs = append(errs, serverCertNotReadyError) + } + + if len(errs) != 0 { + klog.Errorf("hub certificates are not ready: %s", utilerrors.NewAggregate(errs).Error()) + return false + } + return true +} diff --git a/pkg/yurthub/certificate/manager/manager_test.go b/pkg/yurthub/certificate/manager/manager_test.go new file mode 100644 index 00000000000..5a462c2fce4 --- /dev/null +++ b/pkg/yurthub/certificate/manager/manager_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/projectinfo" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" +) + +func TestGetHubServerCertFile(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: "", + path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + "define root dir": { + rootDir: "/tmp", + path: filepath.Join("/tmp", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + opt := &options.YurtHubOptions{ + NodeName: nodeName, + YurtHubHost: "127.0.0.1", + RootDir: tc.rootDir, + } + + mgr, err := NewYurtHubCertManager(opt, remoteServers) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetHubServerCertFile() != tc.path { + t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) + } + }) + } +} + +var ( + joinToken = "123456.abcdef1234567890" + rootDir = "/tmp/token/cert" +) + +func TestReady(t *testing.T) { + nodeName := "foo" + u, _ := url.Parse("http://127.0.0.1") + remoteServers := []*url.URL{u} + + client, err := testdata.CreateCertFakeClient("../testdata") + if err != nil { + t.Errorf("failed to create cert fake client, %v", err) + return + } + + mgr, err := NewYurtHubCertManager(&options.YurtHubOptions{ + NodeName: nodeName, + YurtHubHost: "127.0.0.1", + RootDir: rootDir, + JoinToken: joinToken, + YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, + ClientForTest: client, + }, remoteServers) + if err != nil { + t.Errorf("failed to new yurt cert manager, %v", err) + return + } + mgr.Start() + + err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { + if mgr.Ready() { + return true, nil + } + return false, nil + }) + + if err != nil { + t.Errorf("certificates are not ready, %v", err) + } + + mgr.Stop() + + // reuse the config and ca file + t.Logf("go to check the reuse of config and ca file") + newMgr, err := NewYurtHubCertManager(&options.YurtHubOptions{ + NodeName: nodeName, + YurtHubHost: "127.0.0.1", + RootDir: rootDir, + JoinToken: joinToken, + YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, + ClientForTest: client, + }, remoteServers) + if err != nil { + t.Errorf("failed to new another yurt cert manager, %v", err) + return + } + newMgr.Start() + if !newMgr.Ready() { + t.Errorf("certificates can not be reused") + } + newMgr.Stop() + + os.RemoveAll(rootDir) +} diff --git a/pkg/yurthub/certificate/server/server.go b/pkg/yurthub/certificate/server/server.go new file mode 100644 index 00000000000..eae2f136e9a --- /dev/null +++ b/pkg/yurthub/certificate/server/server.go @@ -0,0 +1,102 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "crypto/tls" + "fmt" + "net" + "time" + + "github.com/pkg/errors" + certificatesv1 "k8s.io/api/certificates/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/authentication/user" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/certificate" + "k8s.io/klog/v2" + + "github.com/openyurtio/openyurt/pkg/projectinfo" + yurtutil "github.com/openyurtio/openyurt/pkg/util" + certfactory "github.com/openyurtio/openyurt/pkg/util/certmanager/factory" + "github.com/openyurtio/openyurt/pkg/util/certmanager/store" + kubeconfigutil "github.com/openyurtio/openyurt/pkg/util/kubeconfig" + hubCert "github.com/openyurtio/openyurt/pkg/yurthub/certificate" +) + +type hubServerCertificateManager struct { + hubServerCertManager certificate.Manager + hubServerCertStore certificate.FileStore +} + +func NewHubServerCertificateManager(client clientset.Interface, clientCertManager hubCert.YurtClientCertificateManager, nodeName, pkiDir string, certIPs []net.IP) (hubCert.YurtServerCertificateManager, error) { + hubServerCertStore, err := store.NewFileStoreWrapper(fmt.Sprintf("%s-server", projectinfo.GetHubName()), pkiDir, pkiDir, "", "") + if err != nil { + return nil, errors.Wrap(err, "couldn't new hub server cert store") + } + + kubeClientFn := func(current *tls.Certificate) (clientset.Interface, error) { + // waiting for the certificate is generated + _ = wait.PollInfinite(5*time.Second, func() (bool, error) { + // keep polling until the yurthub client certificate is signed + if clientCertManager.GetAPIServerClientCert() != nil { + return true, nil + } + klog.Infof("waiting for the controller-manager to sign the %s client certificate", projectinfo.GetHubName()) + return false, nil + }) + + if !yurtutil.IsNil(client) { + return client, nil + } + + return kubeconfigutil.ClientSetFromFile(clientCertManager.GetHubConfFile()) + } + + hubServerCertManager, sErr := certfactory.NewCertManagerFactoryWithFnAndStore(kubeClientFn, hubServerCertStore).New(&certfactory.CertManagerConfig{ + ComponentName: fmt.Sprintf("%s-server", projectinfo.GetHubName()), + SignerName: certificatesv1.KubeletServingSignerName, + ForServerUsage: true, + CommonName: fmt.Sprintf("system:node:%s", nodeName), + Organizations: []string{user.NodesGroup}, + IPs: certIPs, + }) + if sErr != nil { + return nil, sErr + } + + return &hubServerCertificateManager{ + hubServerCertManager: hubServerCertManager, + hubServerCertStore: hubServerCertStore, + }, nil +} + +func (hcm *hubServerCertificateManager) Start() { + hcm.hubServerCertManager.Start() +} + +func (hcm *hubServerCertificateManager) Stop() { + hcm.hubServerCertManager.Stop() +} + +func (hcm *hubServerCertificateManager) GetHubServerCert() *tls.Certificate { + return hcm.hubServerCertManager.Current() +} + +func (hcm *hubServerCertificateManager) GetHubServerCertFile() string { + return hcm.hubServerCertStore.CurrentPath() +} diff --git a/pkg/yurthub/certificate/server/server_test.go b/pkg/yurthub/certificate/server/server_test.go new file mode 100644 index 00000000000..eaa4257f787 --- /dev/null +++ b/pkg/yurthub/certificate/server/server_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2023 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/openyurtio/openyurt/pkg/projectinfo" +) + +func TestGetHubServerCertFile(t *testing.T) { + nodeName := "foo" + testcases := map[string]struct { + rootDir string + path string + }{ + "use default root dir": { + rootDir: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki"), + path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + "define root dir": { + rootDir: "/tmp/pki", + path: filepath.Join("/tmp", "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), + }, + } + + for k, tc := range testcases { + t.Run(k, func(t *testing.T) { + mgr, err := NewHubServerCertificateManager(nil, nil, nodeName, tc.rootDir, nil) + if err != nil { + t.Errorf("failed to new cert manager, %v", err) + } + + if mgr.GetHubServerCertFile() != tc.path { + t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) + } + }) + } +} diff --git a/pkg/yurthub/certificate/token/testdata/ca.crt b/pkg/yurthub/certificate/testdata/ca.crt similarity index 100% rename from pkg/yurthub/certificate/token/testdata/ca.crt rename to pkg/yurthub/certificate/testdata/ca.crt diff --git a/pkg/yurthub/certificate/token/testdata/ca.key b/pkg/yurthub/certificate/testdata/ca.key similarity index 100% rename from pkg/yurthub/certificate/token/testdata/ca.key rename to pkg/yurthub/certificate/testdata/ca.key diff --git a/pkg/yurthub/certificate/token/testdata/fake_client.go b/pkg/yurthub/certificate/testdata/fake_client.go similarity index 100% rename from pkg/yurthub/certificate/token/testdata/fake_client.go rename to pkg/yurthub/certificate/testdata/fake_client.go diff --git a/pkg/yurthub/certificate/testdata/kubelet.conf b/pkg/yurthub/certificate/testdata/kubelet.conf new file mode 100644 index 00000000000..81f572fffa5 --- /dev/null +++ b/pkg/yurthub/certificate/testdata/kubelet.conf @@ -0,0 +1,14 @@ +apiVersion: v1 +clusters: +- cluster: + server: http://127.0.0.1:10261 + name: default-cluster +contexts: +- context: + cluster: default-cluster + namespace: default + user: default-auth + name: default-context +current-context: default-context +kind: Config +preferences: {} \ No newline at end of file diff --git a/pkg/yurthub/certificate/testdata/kubelet.pem b/pkg/yurthub/certificate/testdata/kubelet.pem new file mode 100644 index 00000000000..adb600c9a01 --- /dev/null +++ b/pkg/yurthub/certificate/testdata/kubelet.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIICtDCCAZygAwIBAgIRAO5qKHQa5BXX7iEe6CWQ3pAwDQYJKoZIhvcNAQELBQAw +PjEnMA8GA1UEChMIaGFuZ3pob3UwFAYDVQQKEw1hbGliYWJhIGNsb3VkMRMwEQYD +VQQDEwprdWJlcm5ldGVzMB4XDTIzMDcxOTA2MzMwOFoXDTMzMDcxNjA2MzMwOFow +STEVMBMGA1UEChMMc3lzdGVtOm5vZGVzMTAwLgYDVQQDEydzeXN0ZW06bm9kZTpp +LTV5aHE3YnIwZG5rdjgwcXZmM3hyZnUzYXAwWTATBgcqhkjOPQIBBggqhkjOPQMB +BwNCAARyoup5dNsDp+GOT0nNyowfSp85coVhJ275rqrZOIHIBlhvzJCezK1PVe4r +J9QzzJC03pwl6xoFsFvfI6UG8+G1o20wazAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0l +BAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAfBgNVHSMEGDAWgBQtE/d8pSus +mZzIdzVsFkKmxhhvpjAVBgNVHREEDjAMhwSp/gIBhwR/AAABMA0GCSqGSIb3DQEB +CwUAA4IBAQAoRPEFz1mPq/UzLzSMvxIbmz+FiPH3kuX3/j3FAN+kVelz7MeW/L5/ +HvoxfXWXKm+C0XczNj8Oo2GayNCh4VnHdoWIE2d4XMxZsH1PCJjYHTthJ6WQD1b+ +29VxBQXSthx1WumYkCMDWEduTnTsN3jAbayYBAAWvz+qgBn1Lb/HpJofSCnrZ5je +n596LOHS0UXZDyO5aNVXq0+hydtk/KdR33iA1Tp3X16wqM4C7xeKHixwbYucuUyV +/0P5S0UPp9V03V86xe9p0VHm//1CIS00/wuOh7ituWWX7CU68ZyxyGirAr8r4ojF +wqhGDLZ1ek34ufoADQut/XIhpdeYjNq7 +-----END CERTIFICATE----- +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIJ5xeWx4KNT2gjD4GgWATD19ZSJ00BHQHO61vqYziWm7oAoGCCqGSM49 +AwEHoUQDQgAEcqLqeXTbA6fhjk9JzcqMH0qfOXKFYSdu+a6q2TiByAZYb8yQnsyt +T1XuKyfUM8yQtN6cJesaBbBb3yOlBvPhtQ== +-----END EC PRIVATE KEY----- \ No newline at end of file diff --git a/pkg/yurthub/certificate/token/token.go b/pkg/yurthub/certificate/token/token.go index 6ce163ce2cc..459470e5b41 100644 --- a/pkg/yurthub/certificate/token/token.go +++ b/pkg/yurthub/certificate/token/token.go @@ -28,8 +28,6 @@ import ( "github.com/pkg/errors" certificatesv1 "k8s.io/api/certificates/v1" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/user" clientset "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" @@ -51,39 +49,32 @@ import ( const ( YurtHubCSROrg = "openyurt:yurthub" - DefaultRootDir = "/var/lib" hubPkiDirName = "pki" hubCaFileName = "ca.crt" bootstrapConfigFileName = "bootstrap-hub.conf" ) var ( - hubConfigFileName = fmt.Sprintf("%s.conf", projectinfo.GetHubName()) - serverCertNotReadyError = errors.New("hub server certificate") - apiServerClientCertNotReadyError = errors.New("APIServer client certificate") - caCertIsNotReadyError = errors.New("ca.crt file") + hubConfigFileName = fmt.Sprintf("%s.conf", projectinfo.GetHubName()) ) -type CertificateManagerConfiguration struct { - RootDir string +type ClientCertificateManagerConfiguration struct { + WorkDir string NodeName string JoinToken string BootstrapFile string CaCertHashes []string YurtHubCertOrganizations []string - CertIPs []net.IP RemoteServers []*url.URL Client clientset.Interface } -type yurtHubCertManager struct { +type yurtHubClientCertManager struct { client clientset.Interface remoteServers []*url.URL caCertHashes []string apiServerClientCertManager certificate.Manager - hubServerCertManager certificate.Manager apiServerClientCertStore certificate.FileStore - hubServerCertStore certificate.FileStore hubRunDir string hubName string joinToken string @@ -91,19 +82,13 @@ type yurtHubCertManager struct { dialer *util.Dialer } -// NewYurtHubCertManager new a YurtCertificateManager instance -func NewYurtHubCertManager(cfg *CertificateManagerConfiguration) (hubCert.YurtCertificateManager, error) { +// NewYurtHubClientCertManager new a YurtCertificateManager instance +func NewYurtHubClientCertManager(cfg *ClientCertificateManagerConfiguration) (hubCert.YurtClientCertificateManager, error) { var err error - - hubRunDir := cfg.RootDir - if len(cfg.RootDir) == 0 { - hubRunDir = filepath.Join(DefaultRootDir, projectinfo.GetHubName()) - } - - ycm := &yurtHubCertManager{ + ycm := &yurtHubClientCertManager{ client: cfg.Client, remoteServers: cfg.RemoteServers, - hubRunDir: hubRunDir, + hubRunDir: cfg.WorkDir, hubName: projectinfo.GetHubName(), joinToken: cfg.JoinToken, bootstrapFile: cfg.BootstrapFile, @@ -124,17 +109,6 @@ func NewYurtHubCertManager(cfg *CertificateManagerConfiguration) (hubCert.YurtCe return ycm, errors.Wrap(err, "couldn't new apiserver client certificate manager") } - // 3. prepare yurthub server certificate manager - ycm.hubServerCertStore, err = store.NewFileStoreWrapper(fmt.Sprintf("%s-server", ycm.hubName), ycm.getPkiDir(), ycm.getPkiDir(), "", "") - if err != nil { - return ycm, errors.Wrap(err, "couldn't new hub server cert store") - } - - ycm.hubServerCertManager, err = ycm.newHubServerCertificateManager(ycm.hubServerCertStore, cfg.NodeName, cfg.CertIPs) - if err != nil { - return ycm, errors.Wrap(err, "couldn't new hub server certificate manager") - } - return ycm, nil } @@ -152,7 +126,7 @@ func removeDirContents(dir string) error { return nil } -func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { +func (ycm *yurtHubClientCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { if cfg, err := clientcmd.LoadFromFile(ycm.GetHubConfFile()); err == nil { cluster := kubeconfigutil.GetClusterFromKubeConfig(cfg) if serverURL, err := url.Parse(cluster.Server); err != nil { @@ -172,7 +146,7 @@ func (ycm *yurtHubCertManager) verifyServerAddrOrCleanup(servers []*url.URL) { } // Start init certificate manager and certs for hub agent -func (ycm *yurtHubCertManager) Start() { +func (ycm *yurtHubClientCertManager) Start() { err := ycm.prepareConfigAndCaFile() if err != nil { klog.Errorf("failed to prepare config and ca file, %v", err) @@ -180,7 +154,6 @@ func (ycm *yurtHubCertManager) Start() { } ycm.apiServerClientCertManager.Start() - ycm.hubServerCertManager.Start() } // prepareConfigAndCaFile is used to create the following three files. @@ -188,7 +161,7 @@ func (ycm *yurtHubCertManager) Start() { // - /var/lib/yurthub/yurthub.conf // - /var/lib/yurthub/pki/ca.crt // if these files already exist, just reuse them. -func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error { +func (ycm *yurtHubClientCertManager) prepareConfigAndCaFile() error { var tlsBootstrapCfg *clientcmdapi.Config var hubKubeConfig *clientcmdapi.Config var err error @@ -288,46 +261,23 @@ func (ycm *yurtHubCertManager) prepareConfigAndCaFile() error { } // Stop the cert manager loop -func (ycm *yurtHubCertManager) Stop() { +func (ycm *yurtHubClientCertManager) Stop() { ycm.apiServerClientCertManager.Stop() - ycm.hubServerCertManager.Stop() -} - -// Ready is used for checking client/server/ca certificates are prepared completely or not. -func (ycm *yurtHubCertManager) Ready() bool { - var errs []error - if ycm.GetHubServerCert() == nil { - errs = append(errs, serverCertNotReadyError) - } - - if ycm.GetAPIServerClientCert() == nil { - errs = append(errs, apiServerClientCertNotReadyError) - } - - if exist, _ := util.FileExists(ycm.GetCaFile()); !exist { - errs = append(errs, caCertIsNotReadyError) - } - - if len(errs) != 0 { - klog.Errorf("hub certificates are not ready: %s", utilerrors.NewAggregate(errs).Error()) - return false - } - return true } // UpdateBootstrapConf is used for revising bootstrap conf file by new bearer token. -func (ycm *yurtHubCertManager) UpdateBootstrapConf(joinToken string) error { +func (ycm *yurtHubClientCertManager) UpdateBootstrapConf(joinToken string) error { _, err := ycm.retrieveHubBootstrapConfig(joinToken) return err } // getPkiDir returns the directory for storing hub agent pki -func (ycm *yurtHubCertManager) getPkiDir() string { +func (ycm *yurtHubClientCertManager) getPkiDir() string { return filepath.Join(ycm.hubRunDir, hubPkiDirName) } // getBootstrapConfFile returns the path of yurthub bootstrap conf file -func (ycm *yurtHubCertManager) getBootstrapConfFile() string { +func (ycm *yurtHubClientCertManager) getBootstrapConfFile() string { if len(ycm.bootstrapFile) != 0 { return ycm.bootstrapFile } @@ -335,30 +285,22 @@ func (ycm *yurtHubCertManager) getBootstrapConfFile() string { } // GetCaFile returns the path of ca file -func (ycm *yurtHubCertManager) GetCaFile() string { +func (ycm *yurtHubClientCertManager) GetCaFile() string { return filepath.Join(ycm.getPkiDir(), hubCaFileName) } // GetHubConfFile returns the path of yurtHub config file path -func (ycm *yurtHubCertManager) GetHubConfFile() string { +func (ycm *yurtHubClientCertManager) GetHubConfFile() string { return filepath.Join(ycm.hubRunDir, hubConfigFileName) } -func (ycm *yurtHubCertManager) GetAPIServerClientCert() *tls.Certificate { +func (ycm *yurtHubClientCertManager) GetAPIServerClientCert() *tls.Certificate { return ycm.apiServerClientCertManager.Current() } -func (ycm *yurtHubCertManager) GetHubServerCert() *tls.Certificate { - return ycm.hubServerCertManager.Current() -} - -func (ycm *yurtHubCertManager) GetHubServerCertFile() string { - return ycm.hubServerCertStore.CurrentPath() -} - // newAPIServerClientCertificateManager create a certificate manager for yurthub component to prepare client certificate // that used to proxy requests to remote kube-apiserver. -func (ycm *yurtHubCertManager) newAPIServerClientCertificateManager(fileStore certificate.FileStore, nodeName string, hubCertOrganizations []string) (certificate.Manager, error) { +func (ycm *yurtHubClientCertManager) newAPIServerClientCertificateManager(fileStore certificate.FileStore, nodeName string, hubCertOrganizations []string) (certificate.Manager, error) { orgs := []string{YurtHubCSROrg, user.NodesGroup} for _, v := range hubCertOrganizations { if v != YurtHubCSROrg && v != user.NodesGroup { @@ -374,7 +316,7 @@ func (ycm *yurtHubCertManager) newAPIServerClientCertificateManager(fileStore ce }) } -func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (clientset.Interface, error) { +func (ycm *yurtHubClientCertManager) generateCertClientFn(current *tls.Certificate) (clientset.Interface, error) { var kubeconfig *restclient.Config var err error if !yurtutil.IsNil(ycm.client) { @@ -412,39 +354,7 @@ func (ycm *yurtHubCertManager) generateCertClientFn(current *tls.Certificate) (c return clientset.NewForConfig(kubeconfig) } -// newHubServerCertificateManager create a certificate manager for yurthub component to prepare server certificate -// that used to handle requests from clients on edge nodes. -func (ycm *yurtHubCertManager) newHubServerCertificateManager(fileStore certificate.FileStore, nodeName string, certIPs []net.IP) (certificate.Manager, error) { - kubeClientFn := func(current *tls.Certificate) (clientset.Interface, error) { - // waiting for the certificate is generated - _ = wait.PollInfinite(5*time.Second, func() (bool, error) { - // keep polling until the yurthub client certificate is signed - if ycm.apiServerClientCertManager.Current() != nil { - return true, nil - } - klog.Infof("waiting for the controller-manager to sign the %s client certificate", ycm.hubName) - return false, nil - }) - - if !yurtutil.IsNil(ycm.client) { - return ycm.client, nil - } - - return kubeconfigutil.ClientSetFromFile(ycm.GetHubConfFile()) - } - - // create a certificate manager for the yurthub server and run the csr approver for both yurthub - return certfactory.NewCertManagerFactoryWithFnAndStore(kubeClientFn, fileStore).New(&certfactory.CertManagerConfig{ - ComponentName: fmt.Sprintf("%s-server", ycm.hubName), - SignerName: certificatesv1.KubeletServingSignerName, - ForServerUsage: true, - CommonName: fmt.Sprintf("system:node:%s", nodeName), - Organizations: []string{user.NodesGroup}, - IPs: certIPs, - }) -} - -func (ycm *yurtHubCertManager) retrieveHubBootstrapConfig(joinToken string) (*clientcmdapi.Config, error) { +func (ycm *yurtHubClientCertManager) retrieveHubBootstrapConfig(joinToken string) (*clientcmdapi.Config, error) { // retrieve bootstrap config info from cluster-info configmap by bootstrap token serverAddr := findActiveRemoteServer(ycm.remoteServers).Host if cfg, err := token.RetrieveValidatedConfigInfo(ycm.client, &token.BootstrapData{ diff --git a/pkg/yurthub/certificate/token/token_test.go b/pkg/yurthub/certificate/token/token_test.go index eebc1856340..8a9d07a1ae4 100644 --- a/pkg/yurthub/certificate/token/token_test.go +++ b/pkg/yurthub/certificate/token/token_test.go @@ -18,17 +18,13 @@ package token import ( "fmt" - "net" "net/url" "os" "path/filepath" "testing" - "time" - - "k8s.io/apimachinery/pkg/util/wait" "github.com/openyurtio/openyurt/pkg/projectinfo" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" ) func Test_removeDirContents(t *testing.T) { @@ -78,31 +74,29 @@ func TestGetHubConfFile(t *testing.T) { nodeName := "foo" u, _ := url.Parse("http://127.0.0.1") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} testcases := map[string]struct { - rootDir string + workDir string path string }{ "use default root dir": { - rootDir: "", + workDir: filepath.Join("/var/lib", projectinfo.GetHubName()), path: filepath.Join("/var/lib", projectinfo.GetHubName(), fmt.Sprintf("%s.conf", projectinfo.GetHubName())), }, "define root dir": { - rootDir: "/tmp", + workDir: "/tmp", path: filepath.Join("/tmp", fmt.Sprintf("%s.conf", projectinfo.GetHubName())), }, } for k, tc := range testcases { t.Run(k, func(t *testing.T) { - cfg := &CertificateManagerConfiguration{ + cfg := &ClientCertificateManagerConfiguration{ NodeName: nodeName, RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: tc.rootDir, + WorkDir: tc.workDir, } - mgr, err := NewYurtHubCertManager(cfg) + mgr, err := NewYurtHubClientCertManager(cfg) if err != nil { t.Errorf("failed to new cert manager, %v", err) } @@ -118,31 +112,29 @@ func TestGetCaFile(t *testing.T) { nodeName := "foo" u, _ := url.Parse("http://127.0.0.1") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} testcases := map[string]struct { - rootDir string + workDir string path string }{ "use default root dir": { - rootDir: "", + workDir: filepath.Join("/var/lib", projectinfo.GetHubName()), path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", "ca.crt"), }, "define root dir": { - rootDir: "/tmp", + workDir: "/tmp", path: filepath.Join("/tmp", "pki", "ca.crt"), }, } for k, tc := range testcases { t.Run(k, func(t *testing.T) { - cfg := &CertificateManagerConfiguration{ + cfg := &ClientCertificateManagerConfiguration{ NodeName: nodeName, RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: tc.rootDir, + WorkDir: tc.workDir, } - mgr, err := NewYurtHubCertManager(cfg) + mgr, err := NewYurtHubClientCertManager(cfg) if err != nil { t.Errorf("failed to new cert manager, %v", err) } @@ -154,56 +146,12 @@ func TestGetCaFile(t *testing.T) { } } -func TestGetHubServerCertFile(t *testing.T) { - nodeName := "foo" - u, _ := url.Parse("http://127.0.0.1") - remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} - testcases := map[string]struct { - rootDir string - path string - }{ - "use default root dir": { - rootDir: "", - path: filepath.Join("/var/lib", projectinfo.GetHubName(), "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), - }, - "define root dir": { - rootDir: "/tmp", - path: filepath.Join("/tmp", "pki", fmt.Sprintf("%s-server-current.pem", projectinfo.GetHubName())), - }, - } - - for k, tc := range testcases { - t.Run(k, func(t *testing.T) { - cfg := &CertificateManagerConfiguration{ - NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: tc.rootDir, - } - - mgr, err := NewYurtHubCertManager(cfg) - if err != nil { - t.Errorf("failed to new cert manager, %v", err) - } - - if mgr.GetHubServerCertFile() != tc.path { - t.Errorf("expect hub server cert file %s, but got %s", tc.path, mgr.GetHubServerCertFile()) - } - }) - } -} - -var ( - joinToken = "123456.abcdef1234567890" - rootDir = "/tmp/token/cert" -) - func TestUpdateBootstrapConf(t *testing.T) { + joinToken := "123456.abcdef1234567890" + workDir := "/tmp/token/cert" nodeName := "foo" u, _ := url.Parse("http://127.0.0.1") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} testcases := map[string]struct { joinToken string err error @@ -216,17 +164,16 @@ func TestUpdateBootstrapConf(t *testing.T) { for k, tc := range testcases { t.Run(k, func(t *testing.T) { - client, err := testdata.CreateCertFakeClient("./testdata") + client, err := testdata.CreateCertFakeClient("../testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return } - mgr, err := NewYurtHubCertManager(&CertificateManagerConfiguration{ + mgr, err := NewYurtHubClientCertManager(&ClientCertificateManagerConfiguration{ NodeName: nodeName, RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: rootDir, + WorkDir: workDir, JoinToken: tc.joinToken, Client: client, }) @@ -242,69 +189,5 @@ func TestUpdateBootstrapConf(t *testing.T) { mgr.Stop() }) } - os.RemoveAll(rootDir) -} - -func TestReady(t *testing.T) { - nodeName := "foo" - u, _ := url.Parse("http://127.0.0.1") - remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} - - client, err := testdata.CreateCertFakeClient("./testdata") - if err != nil { - t.Errorf("failed to create cert fake client, %v", err) - return - } - - mgr, err := NewYurtHubCertManager(&CertificateManagerConfiguration{ - NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: rootDir, - JoinToken: joinToken, - YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, - Client: client, - }) - if err != nil { - t.Errorf("failed to new yurt cert manager, %v", err) - return - } - mgr.Start() - - err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { - if mgr.Ready() { - return true, nil - } - return false, nil - }) - - if err != nil { - t.Errorf("certificates are not ready, %v", err) - } - - mgr.Stop() - - // reuse the config and ca file - t.Logf("go to check the reuse of config and ca file") - newMgr, err := NewYurtHubCertManager(&CertificateManagerConfiguration{ - NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, - RootDir: rootDir, - JoinToken: joinToken, - YurtHubCertOrganizations: []string{"yurthub:tenant:foo"}, - Client: client, - }) - if err != nil { - t.Errorf("failed to new another yurt cert manager, %v", err) - return - } - newMgr.Start() - if !newMgr.Ready() { - t.Errorf("certificates can not be reused") - } - newMgr.Stop() - - os.RemoveAll(rootDir) + os.RemoveAll(workDir) } diff --git a/pkg/yurthub/kubernetes/rest/config_test.go b/pkg/yurthub/kubernetes/rest/config_test.go index ee5c450dfe0..bf76a17da04 100644 --- a/pkg/yurthub/kubernetes/rest/config_test.go +++ b/pkg/yurthub/kubernetes/rest/config_test.go @@ -17,7 +17,6 @@ limitations under the License. package rest import ( - "net" "net/url" "os" "testing" @@ -25,8 +24,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" "github.com/openyurtio/openyurt/pkg/yurthub/healthchecker" ) @@ -39,22 +39,20 @@ func TestGetRestConfig(t *testing.T) { servers := map[string]int{"https://10.10.10.113:6443": 2} u, _ := url.Parse("https://10.10.10.113:6443") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} fakeHealthyChecker := healthchecker.NewFakeChecker(false, servers) - client, err := testdata.CreateCertFakeClient("../../certificate/token/testdata") + client, err := testdata.CreateCertFakeClient("../../certificate/testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return } - certManager, err := token.NewYurtHubCertManager(&token.CertificateManagerConfiguration{ + certManager, err := manager.NewYurtHubCertManager(&options.YurtHubOptions{ NodeName: nodeName, - RemoteServers: remoteServers, - CertIPs: certIPs, RootDir: testDir, + YurtHubHost: "127.0.0.1", JoinToken: "123456.abcdef1234567890", - Client: client, - }) + ClientForTest: client, + }, remoteServers) if err != nil { t.Errorf("failed to create certManager, %v", err) return diff --git a/pkg/yurthub/server/certificate_test.go b/pkg/yurthub/server/certificate_test.go index fe2a7a30445..2451f1b057a 100644 --- a/pkg/yurthub/server/certificate_test.go +++ b/pkg/yurthub/server/certificate_test.go @@ -19,7 +19,6 @@ package server import ( "bytes" "encoding/json" - "net" "net/http" "net/http/httptest" "net/url" @@ -29,8 +28,9 @@ import ( "k8s.io/apimachinery/pkg/util/wait" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token" - "github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata" + "github.com/openyurtio/openyurt/cmd/yurthub/app/options" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager" + "github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata" ) var ( @@ -40,20 +40,17 @@ var ( func TestUpdateTokenHandler(t *testing.T) { u, _ := url.Parse("https://10.10.10.113:6443") remoteServers := []*url.URL{u} - certIPs := []net.IP{net.ParseIP("127.0.0.1")} - client, err := testdata.CreateCertFakeClient("../certificate/token/testdata") + client, err := testdata.CreateCertFakeClient("../certificate/testdata") if err != nil { t.Errorf("failed to create cert fake client, %v", err) return } - certManager, err := token.NewYurtHubCertManager(&token.CertificateManagerConfiguration{ + certManager, err := manager.NewYurtHubCertManager(&options.YurtHubOptions{ NodeName: "foo", - RemoteServers: remoteServers, - CertIPs: certIPs, RootDir: testDir, JoinToken: "123456.abcdef1234567890", - Client: client, - }) + ClientForTest: client, + }, remoteServers) if err != nil { t.Errorf("failed to create certManager, %v", err) return @@ -63,7 +60,7 @@ func TestUpdateTokenHandler(t *testing.T) { defer os.RemoveAll(testDir) err = wait.PollImmediate(2*time.Second, 1*time.Minute, func() (done bool, err error) { - if certManager.Ready() { + if certManager.GetAPIServerClientCert() != nil { return true, nil } return false, nil