diff --git a/README.md b/README.md index 248a7fe..f84b117 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,11 @@ A Kubernetes in Kubernetes tool, k3k provides a way to run multiple embedded isolated k3s clusters on your kubernetes cluster. -## Why? +## Example -![alt text](https://github.com/galal-hussein/k3k/blob/main/hack/becausewecan.jpg?raw=true) +An example on creating a k3k cluster on an RKE2 host using k3kcli + +[![asciicast](https://asciinema.org/a/eYlc3dsL2pfP2B50i3Ea8MJJp.svg)](https://asciinema.org/a/eYlc3dsL2pfP2B50i3Ea8MJJp) ## Usage diff --git a/charts/k3k/crds/cluster.yaml b/charts/k3k/crds/cluster.yaml index eb04ce6..e583346 100644 --- a/charts/k3k/crds/cluster.yaml +++ b/charts/k3k/crds/cluster.yaml @@ -39,6 +39,10 @@ spec: type: array items: type: string + tlsSANs: + type: array + items: + type: string expose: type: object properties: @@ -54,6 +58,11 @@ spec: properties: enabled: type: boolean + nodePort: + type: object + properties: + enabled: + type: boolean status: type: object properties: diff --git a/cli/cmds/cluster/create.go b/cli/cmds/cluster/create.go index fc02ff8..ef9e5e3 100644 --- a/cli/cmds/cluster/create.go +++ b/cli/cmds/cluster/create.go @@ -3,21 +3,40 @@ package cluster import ( "context" "errors" + "fmt" + "net/url" "os" + "path/filepath" + "strings" + "time" "github.com/galal-hussein/k3k/cli/cmds" "github.com/galal-hussein/k3k/pkg/apis/k3k.io/v1alpha1" + "github.com/galal-hussein/k3k/pkg/controller/util" "github.com/sirupsen/logrus" "github.com/urfave/cli" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" ) var ( - Scheme = runtime.NewScheme() + Scheme = runtime.NewScheme() + backoff = wait.Backoff{ + Steps: 5, + Duration: 3 * time.Second, + Factor: 2, + Jitter: 0.1, + } ) func init() { @@ -101,10 +120,11 @@ func createCluster(clx *cli.Context) error { ctrlClient, err := client.New(restConfig, client.Options{ Scheme: Scheme, }) + if err != nil { return err } - logrus.Infof("creating a new cluster [%s]", name) + logrus.Infof("Creating a new cluster [%s]", name) cluster := newCluster( name, token, @@ -116,7 +136,54 @@ func createCluster(clx *cli.Context) error { agentArgs, ) - return ctrlClient.Create(ctx, cluster) + cluster.Spec.Expose = &v1alpha1.ExposeConfig{ + NodePort: &v1alpha1.NodePortConfig{ + Enabled: true, + }, + } + + // add Host IP address as an extra TLS-SAN to expose the k3k cluster + url, err := url.Parse(restConfig.Host) + if err != nil { + return err + } + host := strings.Split(url.Host, ":") + cluster.Spec.TLSSANs = []string{ + host[0], + } + + if err := ctrlClient.Create(ctx, cluster); err != nil { + if apierrors.IsAlreadyExists(err) { + logrus.Infof("Cluster [%s] already exists", name) + } else { + return err + } + } + + logrus.Infof("Extracting Kubeconfig for [%s] cluster", name) + var kubeconfig []byte + err = retry.OnError(backoff, apierrors.IsNotFound, func() error { + kubeconfig, err = extractKubeconfig(ctx, ctrlClient, cluster, host[0]) + if err != nil { + return err + } + return nil + }) + + if err != nil { + return err + } + + pwd, err := os.Getwd() + if err != nil { + return err + } + logrus.Infof(`You can start using the cluster with: + + export KUBECONFIG=%s + kubectl cluster-info + `, filepath.Join(pwd, cluster.Name+"-kubeconfig.yaml")) + return os.WriteFile(cluster.Name+"-kubeconfig.yaml", kubeconfig, 0644) } func validateCreateFlags(clx *cli.Context) error { @@ -157,3 +224,78 @@ func newCluster(name, token string, servers, agents int32, clusterCIDR, serviceC }, } } + +func extractKubeconfig(ctx context.Context, client client.Client, cluster *v1alpha1.Cluster, serverIP string) ([]byte, error) { + nn := types.NamespacedName{ + Name: cluster.Name + "-kubeconfig", + Namespace: util.ClusterNamespace(cluster), + } + var kubeSecret v1.Secret + if err := client.Get(ctx, nn, &kubeSecret); err != nil { + return nil, err + } + + kubeconfig := kubeSecret.Data["kubeconfig.yaml"] + if kubeconfig == nil { + return nil, errors.New("empty kubeconfig") + } + + nn = types.NamespacedName{ + Name: "k3k-server-service", + Namespace: util.ClusterNamespace(cluster), + } + var k3kService v1.Service + if err := client.Get(ctx, nn, &k3kService); err != nil { + return nil, err + } + if k3kService.Spec.Type == v1.ServiceTypeNodePort { + nodePort := k3kService.Spec.Ports[0].NodePort + + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfig) + if err != nil { + return nil, err + } + hostURL := fmt.Sprintf("https://%s:%d", serverIP, nodePort) + restConfig.Host = hostURL + + clientConfig := generateKubeconfigFromRest(restConfig) + + b, err := clientcmd.Write(clientConfig) + if err != nil { + return nil, err + } + kubeconfig = b + } + return kubeconfig, nil +} + +func generateKubeconfigFromRest(config *rest.Config) clientcmdapi.Config { + clusters := make(map[string]*clientcmdapi.Cluster) + clusters["default-cluster"] = &clientcmdapi.Cluster{ + Server: config.Host, + CertificateAuthorityData: config.CAData, + } + + contexts := make(map[string]*clientcmdapi.Context) + contexts["default-context"] = &clientcmdapi.Context{ + Cluster: "default-cluster", + Namespace: "default", + AuthInfo: "default", + } + + authinfos := make(map[string]*clientcmdapi.AuthInfo) + authinfos["default"] = &clientcmdapi.AuthInfo{ + ClientCertificateData: config.CertData, + ClientKeyData: config.KeyData, + } + + clientConfig := clientcmdapi.Config{ + Kind: "Config", + APIVersion: "v1", + Clusters: clusters, + Contexts: contexts, + CurrentContext: "default-context", + AuthInfos: authinfos, + } + return clientConfig +} diff --git a/pkg/apis/k3k.io/v1alpha1/types.go b/pkg/apis/k3k.io/v1alpha1/types.go index 76e3707..e0596ba 100644 --- a/pkg/apis/k3k.io/v1alpha1/types.go +++ b/pkg/apis/k3k.io/v1alpha1/types.go @@ -16,17 +16,17 @@ type Cluster struct { } type ClusterSpec struct { - Name string `json:"name"` - Version string `json:"version"` - Servers *int32 `json:"servers"` - Agents *int32 `json:"agents"` - Token string `json:"token"` - ClusterCIDR string `json:"clusterCIDR,omitempty"` - ServiceCIDR string `json:"serviceCIDR,omitempty"` - ClusterDNS string `json:"clusterDNS,omitempty"` - - ServerArgs []string `json:"serverArgs,omitempty"` - AgentArgs []string `json:"agentArgs,omitempty"` + Name string `json:"name"` + Version string `json:"version"` + Servers *int32 `json:"servers"` + Agents *int32 `json:"agents"` + Token string `json:"token"` + ClusterCIDR string `json:"clusterCIDR,omitempty"` + ServiceCIDR string `json:"serviceCIDR,omitempty"` + ClusterDNS string `json:"clusterDNS,omitempty"` + ServerArgs []string `json:"serverArgs,omitempty"` + AgentArgs []string `json:"agentArgs,omitempty"` + TLSSANs []string `json:"tlsSANs,omitempty"` Expose *ExposeConfig `json:"expose,omitempty"` } @@ -43,6 +43,7 @@ type ClusterList struct { type ExposeConfig struct { Ingress *IngressConfig `json:"ingress"` LoadBalancer *LoadBalancerConfig `json:"loadbalancer"` + NodePort *NodePortConfig `json:"nodePort"` } type IngressConfig struct { @@ -54,6 +55,10 @@ type LoadBalancerConfig struct { Enabled bool `json:"enabled"` } +type NodePortConfig struct { + Enabled bool `json:"enabled"` +} + type ClusterStatus struct { ClusterCIDR string `json:"clusterCIDR,omitempty"` ServiceCIDR string `json:"serviceCIDR,omitempty"` diff --git a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go index 06d5e04..6827ab0 100644 --- a/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/k3k.io/v1alpha1/zz_generated.deepcopy.go @@ -207,6 +207,11 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.TLSSANs != nil { + in, out := &in.TLSSANs, &out.TLSSANs + *out = make([]string, len(*in)) + copy(*out, *in) + } if in.Expose != nil { in, out := &in.Expose, &out.Expose *out = new(ExposeConfig) @@ -254,6 +259,11 @@ func (in *ExposeConfig) DeepCopyInto(out *ExposeConfig) { *out = new(LoadBalancerConfig) **out = **in } + if in.NodePort != nil { + in, out := &in.NodePort, &out.NodePort + *out = new(NodePortConfig) + **out = **in + } return } @@ -298,3 +308,19 @@ func (in *LoadBalancerConfig) DeepCopy() *LoadBalancerConfig { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePortConfig) DeepCopyInto(out *NodePortConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePortConfig. +func (in *NodePortConfig) DeepCopy() *NodePortConfig { + if in == nil { + return nil + } + out := new(NodePortConfig) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/controller/cluster/config/server.go b/pkg/controller/cluster/config/server.go index eadcaf2..3c0e650 100644 --- a/pkg/controller/cluster/config/server.go +++ b/pkg/controller/cluster/config/server.go @@ -56,6 +56,13 @@ func serverOptions(cluster *v1alpha1.Cluster) string { if cluster.Spec.ClusterDNS != "" { opts = opts + "cluster-dns: " + cluster.Spec.ClusterDNS + "\n" } + if len(cluster.Spec.TLSSANs) > 0 { + opts = opts + "tls-san:\n" + for _, addr := range cluster.Spec.TLSSANs { + opts = opts + "- " + addr + "\n" + } + } + // TODO: Add extra args to the options return opts } diff --git a/pkg/controller/cluster/server/service.go b/pkg/controller/cluster/server/service.go index 56e7de3..d312ef9 100644 --- a/pkg/controller/cluster/server/service.go +++ b/pkg/controller/cluster/server/service.go @@ -8,6 +8,14 @@ import ( ) func Service(cluster *v1alpha1.Cluster) *v1.Service { + serviceType := v1.ServiceTypeClusterIP + if cluster.Spec.Expose != nil { + if cluster.Spec.Expose.NodePort != nil { + if cluster.Spec.Expose.NodePort.Enabled { + serviceType = v1.ServiceTypeNodePort + } + } + } return &v1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -18,7 +26,7 @@ func Service(cluster *v1alpha1.Cluster) *v1.Service { Namespace: util.ClusterNamespace(cluster), }, Spec: v1.ServiceSpec{ - Type: v1.ServiceTypeClusterIP, + Type: serviceType, Selector: map[string]string{ "cluster": cluster.Name, "role": "server",