diff --git a/phase/configure_k0s.go b/phase/configure_k0s.go index 26163032..4f252ead 100644 --- a/phase/configure_k0s.go +++ b/phase/configure_k0s.go @@ -183,7 +183,7 @@ func (p *ConfigureK0s) generateDefaultConfig() (string, error) { func (p *ConfigureK0s) Run() error { controllers := p.Config.Spec.Hosts.Controllers().Filter(func(h *cluster.Host) bool { return !h.Reset && len(h.Metadata.K0sNewConfig) > 0 - }) + }) return p.parallelDo(controllers, p.configureK0s) } @@ -270,19 +270,6 @@ func (p *ConfigureK0s) configureK0s(h *cluster.Host) error { return nil } -func addUnlessExist(slice *[]string, s string) { - var found bool - for _, v := range *slice { - if v == s { - found = true - break - } - } - if !found { - *slice = append(*slice, s) - } -} - func (p *ConfigureK0s) configFor(h *cluster.Host) (string, error) { var cfg dig.Mapping @@ -298,40 +285,12 @@ func (p *ConfigureK0s) configFor(h *cluster.Host) (string, error) { cfg = p.newBaseConfig.Dup() } - var sans []string - var addr string if h.PrivateAddress != "" { addr = h.PrivateAddress } else { addr = h.Address() } - cfg.DigMapping("spec", "api")["address"] = addr - addUnlessExist(&sans, addr) - - oldsans := cfg.Dig("spec", "api", "sans") - switch oldsans := oldsans.(type) { - case []interface{}: - for _, v := range oldsans { - if s, ok := v.(string); ok { - addUnlessExist(&sans, s) - } - } - case []string: - for _, v := range oldsans { - addUnlessExist(&sans, v) - } - } - - var controllers cluster.Hosts = p.Config.Spec.Hosts.Controllers() - for _, c := range controllers { - addUnlessExist(&sans, c.Address()) - if c.PrivateAddress != "" { - addUnlessExist(&sans, c.PrivateAddress) - } - } - addUnlessExist(&sans, "127.0.0.1") - cfg.DigMapping("spec", "api")["sans"] = sans if cfg.Dig("spec", "storage", "etcd", "peerAddress") != nil || h.PrivateAddress != "" { cfg.DigMapping("spec", "storage", "etcd")["peerAddress"] = addr diff --git a/phase/get_kubeconfig.go b/phase/get_kubeconfig.go index 19943fe9..ced829fb 100644 --- a/phase/get_kubeconfig.go +++ b/phase/get_kubeconfig.go @@ -2,13 +2,10 @@ package phase import ( "fmt" - "strings" "github.com/alessio/shellescape" - "github.com/k0sproject/dig" "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster" "github.com/k0sproject/rig/exec" - "gopkg.in/yaml.v2" "k8s.io/client-go/tools/clientcmd" ) @@ -31,24 +28,6 @@ var readKubeconfig = func(h *cluster.Host) (string, error) { return output, nil } -var k0sConfig = func(h *cluster.Host) (dig.Mapping, error) { - cfgContent, err := h.Configurer.ReadFile(h, h.Configurer.K0sConfigPath()) - if err != nil { - return nil, fmt.Errorf("read k0s config from host: %w", err) - } - - var cfg dig.Mapping - if err := yaml.Unmarshal([]byte(cfgContent), &cfg); err != nil { - return nil, fmt.Errorf("unmarshal k0s config: %w", err) - } - - if err != nil { - return nil, fmt.Errorf("parse k0s config: %w", err) - } - - return cfg, nil -} - func (p *GetKubeconfig) DryRun() error { p.DryMsg(p.Config.Spec.Hosts.Controllers()[0], "get admin kubeconfig") return nil @@ -58,34 +37,13 @@ func (p *GetKubeconfig) DryRun() error { func (p *GetKubeconfig) Run() error { h := p.Config.Spec.Hosts.Controllers()[0] - cfg, err := k0sConfig(h) - if err != nil { - return err - } - output, err := readKubeconfig(h) if err != nil { return fmt.Errorf("read kubeconfig from host: %w", err) } if p.APIAddress == "" { - // the controller admin.conf is aways pointing to localhost, thus we need to change the address - // something usable from outside - address := h.Address() - if a, ok := cfg.Dig("spec", "api", "externalAddress").(string); ok && a != "" { - address = a - } - - port := 6443 - if p, ok := cfg.Dig("spec", "api", "port").(int); ok && p != 0 { - port = p - } - - if strings.Contains(address, ":") { - p.APIAddress = fmt.Sprintf("https://[%s]:%d", address, port) - } else { - p.APIAddress = fmt.Sprintf("https://%s:%d", address, port) - } + p.APIAddress = p.Config.Spec.KubeAPIURL() } cfgString, err := kubeConfig(output, p.Config.Metadata.Name, p.APIAddress) diff --git a/phase/get_kubeconfig_test.go b/phase/get_kubeconfig_test.go index cb42fc64..e2f093fb 100644 --- a/phase/get_kubeconfig_test.go +++ b/phase/get_kubeconfig_test.go @@ -49,12 +49,6 @@ func TestGetKubeconfig(t *testing.T) { defer func() { readKubeconfig = origReadKubeconfig }() readKubeconfig = fakeReader - origK0sConfig := k0sConfig - defer func() { k0sConfig = origK0sConfig }() - k0sConfig = func(h *cluster.Host) (dig.Mapping, error) { - return cfg.Spec.K0s.Config, nil - } - p := GetKubeconfig{GenericPhase: GenericPhase{Config: cfg}} require.NoError(t, p.Run()) conf, err := clientcmd.Load([]byte(cfg.Metadata.Kubeconfig)) diff --git a/phase/initialize_k0s.go b/phase/initialize_k0s.go index 7cdc9ba4..821891ff 100644 --- a/phase/initialize_k0s.go +++ b/phase/initialize_k0s.go @@ -100,7 +100,6 @@ func (p *InitializeK0s) Run() error { } return nil }) - if err != nil { return err } @@ -116,18 +115,13 @@ func (p *InitializeK0s) Run() error { return err } - port := 6443 - if p, ok := p.Config.Spec.K0s.Config.Dig("spec", "api", "port").(int); ok { - port = p - } log.Infof("%s: waiting for kubernetes api to respond", h) - if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.KubeAPIReadyFunc(h, port)); err != nil { + if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.KubeAPIReadyFunc(h, p.Config)); err != nil { return err } return nil }) - if err != nil { return err } diff --git a/phase/install_controllers.go b/phase/install_controllers.go index 5ce0e412..dd91b631 100644 --- a/phase/install_controllers.go +++ b/phase/install_controllers.go @@ -88,7 +88,7 @@ func (p *InstallControllers) After() error { // Run the phase func (p *InstallControllers) Run() error { - url := p.Config.Spec.KubeAPIURL() + url := p.Config.Spec.InternalKubeAPIURL() healthz := fmt.Sprintf("%s/healthz", url) err := p.parallelDo(p.hosts, func(h *cluster.Host) error { @@ -191,11 +191,6 @@ func (p *InstallControllers) Run() error { } func (p *InstallControllers) waitJoined(h *cluster.Host) error { - port := 6443 - if p, ok := p.Config.Spec.K0s.Config.Dig("spec", "api", "port").(int); ok { - port = p - } - log.Infof("%s: waiting for kubernetes api to respond", h) - return retry.Timeout(context.TODO(), retry.DefaultTimeout, node.KubeAPIReadyFunc(h, port)) + return retry.Timeout(context.TODO(), retry.DefaultTimeout, node.KubeAPIReadyFunc(h, p.Config)) } diff --git a/phase/install_workers.go b/phase/install_workers.go index 970d9ec2..ebdc1fbe 100644 --- a/phase/install_workers.go +++ b/phase/install_workers.go @@ -98,7 +98,7 @@ func (p *InstallWorkers) After() error { // Run the phase func (p *InstallWorkers) Run() error { - url := p.Config.Spec.KubeAPIURL() + url := p.Config.Spec.InternalKubeAPIURL() healthz := fmt.Sprintf("%s/healthz", url) err := p.parallelDo(p.hosts, func(h *cluster.Host) error { diff --git a/phase/upgrade_controllers.go b/phase/upgrade_controllers.go index 247ee0b4..df526a8f 100644 --- a/phase/upgrade_controllers.go +++ b/phase/upgrade_controllers.go @@ -131,13 +131,9 @@ func (p *UpgradeControllers) Run() error { if err != nil { return err } - port := 6443 - if p, ok := p.Config.Spec.K0s.Config.Dig("spec", "api", "port").(int); ok { - port = p - } if p.IsWet() { - if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.KubeAPIReadyFunc(h, port)); err != nil { + if err := retry.Timeout(context.TODO(), retry.DefaultTimeout, node.KubeAPIReadyFunc(h, p.Config)); err != nil { return fmt.Errorf("kube api did not become ready: %w", err) } } diff --git a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/spec.go b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/spec.go index 02a6dc49..050be23e 100644 --- a/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/spec.go +++ b/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster/spec.go @@ -2,9 +2,11 @@ package cluster import ( "fmt" + "strings" "github.com/creasty/defaults" "github.com/jellydator/validation" + "github.com/k0sproject/dig" ) // Spec defines cluster config spec section @@ -80,24 +82,71 @@ func (s *Spec) Validate() error { ) } -// KubeAPIURL returns an url to the cluster's kube api -func (s *Spec) KubeAPIURL() string { - var caddr string +func (s *Spec) clusterExternalAddress() string { if a := s.K0s.Config.DigString("spec", "api", "externalAddress"); a != "" { - caddr = a - } else { - leader := s.K0sLeader() - if leader.PrivateAddress != "" { - caddr = leader.PrivateAddress - } else { - caddr = leader.Address() + return a + } + + if cplb, ok := s.K0s.Config.Dig("spec", "network", "controlPlaneLoadBalancing").(dig.Mapping); ok { + if enabled, ok := cplb.Dig("enabled").(bool); ok && enabled { + vrrpAddresses := cplb.Dig("virtualServers").([]string) + if len(vrrpAddresses) > 0 { + return vrrpAddresses[0] + } } } - cport := 6443 + return s.K0sLeader().Address() +} + +func (s *Spec) clusterInternalAddress() string { + leader := s.K0sLeader() + if leader.PrivateAddress != "" { + return leader.PrivateAddress + } else { + return leader.Address() + } +} + +const defaultAPIPort = 6443 + +func (s *Spec) APIPort() int { if p, ok := s.K0s.Config.Dig("spec", "api", "port").(int); ok { - cport = p + return p } + return defaultAPIPort +} - return fmt.Sprintf("https://%s:%d", caddr, cport) +// KubeAPIURL returns an external url to the cluster's kube API +func (s *Spec) KubeAPIURL() string { + return fmt.Sprintf("https://%s:%d", formatIPV6(s.clusterExternalAddress()), s.APIPort()) +} + +// InternalKubeAPIURL returns a cluster internal url to the cluster's kube API +func (s *Spec) InternalKubeAPIURL() string { + return fmt.Sprintf("https://%s:%d", formatIPV6(s.clusterInternalAddress()), s.APIPort()) +} + +// NodeInternalKubeAPIURL returns a cluster internal url to the node's kube API +func (s *Spec) NodeInternalKubeAPIURL(h *Host) string { + addr := "127.0.0.1" + + // spec.api.onlyBindToAddress was introduced in k0s 1.30. Setting it to true will make the API server only + // listen on the IP address configured by the `address` option. + if onlyBindAddr, ok := s.K0s.Config.Dig("spec", "api", "onlyBindToAddress").(bool); ok && onlyBindAddr { + if h.PrivateAddress != "" { + addr = h.PrivateAddress + } else { + addr = h.Address() + } + } + + return fmt.Sprintf("https://%s:%d", formatIPV6(addr), s.APIPort()) +} + +func formatIPV6(address string) string { + if strings.Contains(address, ":") { + return fmt.Sprintf("[%s]", address) + } + return address } diff --git a/pkg/node/statusfunc.go b/pkg/node/statusfunc.go index 3066b56d..67d16af0 100644 --- a/pkg/node/statusfunc.go +++ b/pkg/node/statusfunc.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1" "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster" "github.com/k0sproject/rig/exec" @@ -195,8 +196,8 @@ func ServiceStoppedFunc(h *cluster.Host, service string) retryFunc { } // KubeAPIReadyFunc returns a function that returns an error unless the host's local kube api responds to /version -func KubeAPIReadyFunc(h *cluster.Host, port int) retryFunc { +func KubeAPIReadyFunc(h *cluster.Host, config *v1beta1.Cluster) retryFunc { // If the anon-auth is disabled on kube api the version endpoint will give 401 // thus we need to accept both 200 and 401 as valid statuses when checking kube api - return HTTPStatusFunc(h, fmt.Sprintf("https://localhost:%d/version", port), 200, 401) + return HTTPStatusFunc(h, fmt.Sprintf("%s/version", config.Spec.NodeInternalKubeAPIURL(h)), 200, 401) }