Skip to content

Commit

Permalink
add kubelet certificate mode in yurthub (#1625)
Browse files Browse the repository at this point in the history
  • Loading branch information
rambohe-ch committed Jul 24, 2023
1 parent fe46a9d commit 4efa1bf
Show file tree
Hide file tree
Showing 20 changed files with 733 additions and 315 deletions.
57 changes: 14 additions & 43 deletions cmd/yurthub/app/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,9 @@ import (

"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
"github.com/openyurtio/openyurt/pkg/projectinfo"
ipUtils "github.com/openyurtio/openyurt/pkg/util/ip"
"github.com/openyurtio/openyurt/pkg/yurthub/cachemanager"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token"
certificatemgr "github.com/openyurtio/openyurt/pkg/yurthub/certificate/manager"
"github.com/openyurtio/openyurt/pkg/yurthub/filter"
"github.com/openyurtio/openyurt/pkg/yurthub/filter/manager"
"github.com/openyurtio/openyurt/pkg/yurthub/kubernetes/meta"
Expand Down Expand Up @@ -178,10 +177,21 @@ func Complete(options *options.YurtHubOptions) (*YurtHubConfiguration, error) {
LeaderElection: options.LeaderElection,
}

certMgr, err := createCertManager(options, us)
certMgr, err := certificatemgr.NewYurtHubCertManager(options, us)
if err != nil {
return nil, err
}
certMgr.Start()
err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) {
isReady := certMgr.Ready()
if isReady {
return true, nil
}
return false, nil
})
if err != nil {
return nil, fmt.Errorf("hub certificates preparation failed, %v", err)
}
cfg.CertManager = certMgr

if options.EnableDummyIf {
Expand Down Expand Up @@ -230,7 +240,7 @@ func parseRemoteServers(serverAddr string) ([]*url.URL, error) {
return us, nil
}

// createSharedInformers create sharedInformers from the given proxyAddr.
// createClientAndSharedInformers create kubeclient and sharedInformers from the given proxyAddr.
func createClientAndSharedInformers(proxyAddr string, enableNodePool bool) (kubernetes.Interface, informers.SharedInformerFactory, yurtinformers.SharedInformerFactory, error) {
var kubeConfig *rest.Config
var yurtClient yurtclientset.Interface
Expand Down Expand Up @@ -341,45 +351,6 @@ func isServiceTopologyFilterEnabled(options *options.YurtHubOptions) bool {
return true
}

func createCertManager(options *options.YurtHubOptions, remoteServers []*url.URL) (certificate.YurtCertificateManager, error) {
// use dummy ip and bind ip as cert IP SANs
certIPs := ipUtils.RemoveDupIPs([]net.IP{
net.ParseIP(options.HubAgentDummyIfIP),
net.ParseIP(options.YurtHubHost),
net.ParseIP(options.YurtHubProxyHost),
})

cfg := &token.CertificateManagerConfiguration{
RootDir: options.RootDir,
NodeName: options.NodeName,
JoinToken: options.JoinToken,
BootstrapFile: options.BootstrapFile,
CaCertHashes: options.CACertHashes,
YurtHubCertOrganizations: options.YurtHubCertOrganizations,
CertIPs: certIPs,
RemoteServers: remoteServers,
Client: options.ClientForTest,
}
certManager, err := token.NewYurtHubCertManager(cfg)
if err != nil {
return nil, fmt.Errorf("failed to create cert manager for yurthub, %v", err)
}

certManager.Start()
err = wait.PollImmediate(5*time.Second, 4*time.Minute, func() (bool, error) {
isReady := certManager.Ready()
if isReady {
return true, nil
}
return false, nil
})
if err != nil {
return nil, fmt.Errorf("hub certificates preparation failed, %v", err)
}

return certManager, nil
}

func prepareServerServing(options *options.YurtHubOptions, certMgr certificate.YurtCertificateManager, cfg *YurtHubConfiguration) error {
if err := (&apiserveroptions.DeprecatedInsecureServingOptions{
BindAddress: net.ParseIP(options.YurtHubHost),
Expand Down
4 changes: 2 additions & 2 deletions cmd/yurthub/app/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ import (
"testing"

"github.com/openyurtio/openyurt/cmd/yurthub/app/options"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/token/testdata"
"github.com/openyurtio/openyurt/pkg/yurthub/certificate/testdata"
)

func TestComplete(t *testing.T) {
options := options.NewYurtHubOptions()
client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/token/testdata")
client, err := testdata.CreateCertFakeClient("../../../../pkg/yurthub/certificate/testdata")
if err != nil {
t.Errorf("failed to create cert fake client, %v", err)
return
Expand Down
3 changes: 3 additions & 0 deletions cmd/yurthub/app/options/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type YurtHubOptions struct {
HeartbeatIntervalSeconds int
MaxRequestInFlight int
JoinToken string
BootstrapMode string
BootstrapFile string
RootDir string
Version bool
Expand Down Expand Up @@ -105,6 +106,7 @@ func NewYurtHubOptions() *YurtHubOptions {
HeartbeatTimeoutSeconds: 2,
HeartbeatIntervalSeconds: 10,
MaxRequestInFlight: 250,
BootstrapMode: "token",
RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()),
EnableProfiling: true,
EnableDummyIf: true,
Expand Down Expand Up @@ -189,6 +191,7 @@ func (o *YurtHubOptions) AddFlags(fs *pflag.FlagSet) {
fs.IntVar(&o.MaxRequestInFlight, "max-requests-in-flight", o.MaxRequestInFlight, "the maximum number of parallel requests.")
fs.StringVar(&o.JoinToken, "join-token", o.JoinToken, "the Join token for bootstrapping hub agent.")
fs.MarkDeprecated("join-token", "It is planned to be removed from OpenYurt in the version v1.5. Please use --bootstrap-file to bootstrap hub agent.")
fs.StringVar(&o.BootstrapMode, "bootstrap-mode", o.BootstrapMode, "the mode for bootstrapping hub agent(token, kubeletcertificate).")
fs.StringVar(&o.BootstrapFile, "bootstrap-file", o.BootstrapFile, "the bootstrap file for bootstrapping hub agent.")
fs.StringVar(&o.RootDir, "root-dir", o.RootDir, "directory path for managing hub agent files(pki, cache etc).")
fs.BoolVar(&o.Version, "version", o.Version, "print the version information.")
Expand Down
1 change: 1 addition & 0 deletions cmd/yurthub/app/options/options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ func TestNewYurtHubOptions(t *testing.T) {
HeartbeatTimeoutSeconds: 2,
HeartbeatIntervalSeconds: 10,
MaxRequestInFlight: 250,
BootstrapMode: "token",
RootDir: filepath.Join("/var/lib/", projectinfo.GetHubName()),
EnableProfiling: true,
EnableDummyIf: true,
Expand Down
15 changes: 13 additions & 2 deletions pkg/yurthub/certificate/interfaces.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,25 @@ import (

// YurtCertificateManager is responsible for managing node certificate for yurthub
type YurtCertificateManager interface {
Start()
Stop()
YurtClientCertificateManager
YurtServerCertificateManager
// Ready should be called after yurt certificate manager started by Start.
Ready() bool
}

// YurtClientCertificateManager is responsible for managing node client certificates for yurthub
type YurtClientCertificateManager interface {
Start()
Stop()
UpdateBootstrapConf(joinToken string) error
GetHubConfFile() string
GetCaFile() string
GetAPIServerClientCert() *tls.Certificate
}

type YurtServerCertificateManager interface {
Start()
Stop()
GetHubServerCert() *tls.Certificate
GetHubServerCertFile() string
}
119 changes: 119 additions & 0 deletions pkg/yurthub/certificate/kubeletcertificate/kubelet_certificate.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
Copyright 2023 The OpenYurt Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubeletcertificate

import (
"crypto/tls"
"crypto/x509"
"errors"
"fmt"
"time"

"k8s.io/klog/v2"

"github.com/openyurtio/openyurt/pkg/yurthub/certificate"
"github.com/openyurtio/openyurt/pkg/yurthub/util"
)

var (
KubeConfNotExistErr = errors.New("/etc/kubernetes/kubelet.conf file doesn't exist")
KubeletCANotExistErr = errors.New("/etc/kubernetes/pki/ca.crt file doesn't exist")
KubeletPemNotExistErr = errors.New("/var/lib/kubelet/pki/kubelet-current.pem file doesn't exist")
)

type kubeletCertManager struct {
kubeConfFile string
kubeletCAFile string
kubeletPemFile string
cert *tls.Certificate
}

func NewKubeletCertManager(kubeConfFile, kubeletCAFile, kubeletPemFile string) (certificate.YurtClientCertificateManager, error) {
if exist, _ := util.FileExists(kubeConfFile); !exist {
return nil, KubeConfNotExistErr
}

if exist, _ := util.FileExists(kubeletCAFile); !exist {
return nil, KubeletCANotExistErr
}

if exist, _ := util.FileExists(kubeletPemFile); !exist {
return nil, KubeletPemNotExistErr
}

cert, err := loadFile(kubeletPemFile)
if err != nil {
return nil, err
}

return &kubeletCertManager{
kubeConfFile: kubeConfFile,
kubeletCAFile: kubeletCAFile,
kubeletPemFile: kubeletPemFile,
cert: cert,
}, nil
}

func (kcm *kubeletCertManager) Start() {
// do nothing
}

func (kcm *kubeletCertManager) Stop() {
// do nothing
}

func (kcm *kubeletCertManager) UpdateBootstrapConf(_ string) error {
return nil
}

func (kcm *kubeletCertManager) GetHubConfFile() string {
return kcm.kubeConfFile
}

func (kcm *kubeletCertManager) GetCaFile() string {
return kcm.kubeletCAFile
}

func (kcm *kubeletCertManager) GetAPIServerClientCert() *tls.Certificate {
if kcm.cert != nil && kcm.cert.Leaf != nil && !time.Now().After(kcm.cert.Leaf.NotAfter) {
return kcm.cert
}

klog.Warningf("current certificate: %s is expired, reload it", kcm.kubeletPemFile)
cert, err := loadFile(kcm.kubeletPemFile)
if err != nil {
klog.Errorf("failed to load client certificate(%s), %v", kcm.kubeletPemFile, err)
return nil
}
kcm.cert = cert
return kcm.cert
}

func loadFile(pairFile string) (*tls.Certificate, error) {
// LoadX509KeyPair knows how to parse combined cert and private key from
// the same file.
cert, err := tls.LoadX509KeyPair(pairFile, pairFile)
if err != nil {
return nil, fmt.Errorf("could not convert data from %q into cert/key pair: %v", pairFile, err)
}
certs, err := x509.ParseCertificates(cert.Certificate[0])
if err != nil {
return nil, fmt.Errorf("unable to parse certificate data: %v", err)
}
cert.Leaf = certs[0]
return &cert, nil
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
Copyright 2023 The OpenYurt Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubeletcertificate

import "testing"

func TestNewKubeletCertManager(t *testing.T) {
testcases := map[string]struct {
kubeConfFile string
kubeletCAFile string
kubeletPemFile string
err error
}{
"kubelet.conf doesn't exist": {
kubeConfFile: "invalid file",
err: KubeConfNotExistErr,
},
"ca.crt file doesn't exist": {
kubeConfFile: "../testdata/kubelet.conf",
kubeletCAFile: "invalid file",
err: KubeletCANotExistErr,
},
"kubelet.pem doesn't exist": {
kubeConfFile: "../testdata/kubelet.conf",
kubeletCAFile: "../testdata/ca.crt",
kubeletPemFile: "invalid file",
err: KubeletPemNotExistErr,
},
"normal kubelet cert manager": {
kubeConfFile: "../testdata/kubelet.conf",
kubeletCAFile: "../testdata/ca.crt",
kubeletPemFile: "../testdata/kubelet.pem",
err: nil,
},
}

for k, tc := range testcases {
t.Run(k, func(t *testing.T) {
_, err := NewKubeletCertManager(tc.kubeConfFile, tc.kubeletCAFile, tc.kubeletPemFile)
if err != tc.err {
t.Errorf("expect error is %v, but got %v", tc.err, err)
}
})
}
}
Loading

0 comments on commit 4efa1bf

Please sign in to comment.