Merge pull request eksctl-io#7218 from cPu1/cluster-subnets-security-…

…groups

Allow mutating control plane subnets and security groups

examples/38-cluster-subnets-sgs.yaml

@@ -0,0 +1,22 @@
+# An example config for updating API server endpoint access, public access CIDRs, and control plane subnets and security groups.
+# To perform the update, run `eksctl utils update-cluster-vpc-config -f 38-cluster-subnets-sgs.yaml`
+
+apiVersion: eksctl.io/v1alpha5
+kind: ClusterConfig
+metadata:
+  name: cluster-38
+  region: us-west-2
+
+iam:
+  withOIDC: true
+
+vpc:
+  controlPlaneSubnetIDs: [subnet-1234, subnet-5678]
+  controlPlaneSecurityGroupIDs: [sg-1234, sg-5678]
+  clusterEndpoints:
+    publicAccess: true
+    privateAccess: true
+  publicAccessCIDRs: ["1.1.1.1/32"]
+
+managedNodeGroups:
+  - name: mng1

integration/tests/managed/managed_nodegroup_test.go

@@ -6,10 +6,13 @@ package managed
 import (
 	"context"
 	"fmt"
+	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/ec2"
+	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	awseks "github.com/aws/aws-sdk-go-v2/service/eks"
 
 	harness "github.com/dlespiau/kube-test-harness"
@@ -27,6 +30,7 @@ import (
 	clusterutils "github.com/weaveworks/eksctl/integration/utilities/cluster"
 	"github.com/weaveworks/eksctl/integration/utilities/kube"
 	api "github.com/weaveworks/eksctl/pkg/apis/eksctl.io/v1alpha5"
+	"github.com/weaveworks/eksctl/pkg/awsapi"
 	"github.com/weaveworks/eksctl/pkg/eks"
 	"github.com/weaveworks/eksctl/pkg/testutils"
 )
@@ -521,6 +525,107 @@ var _ = Describe("(Integration) Create Managed Nodegroups", func() {
 			Expect(cmd).To(RunSuccessfully())
 		})
 	})
+
+	Context("eksctl utils update-cluster-vpc-config", Serial, func() {
+		makeAWSProvider := func(ctx context.Context, clusterConfig *api.ClusterConfig) api.ClusterProvider {
+			clusterProvider, err := eks.New(ctx, &api.ProviderConfig{Region: params.Region}, clusterConfig)
+			Expect(err).NotTo(HaveOccurred())
+			return clusterProvider.AWSProvider
+		}
+		getPrivateSubnetIDs := func(ctx context.Context, ec2API awsapi.EC2, vpcID string) []string {
+			out, err := ec2API.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{
+				Filters: []ec2types.Filter{
+					{
+						Name:   aws.String("vpc-id"),
+						Values: []string{vpcID},
+					},
+				},
+			})
+			Expect(err).NotTo(HaveOccurred())
+			var subnetIDs []string
+			for _, s := range out.Subnets {
+				if !*s.MapPublicIpOnLaunch {
+					subnetIDs = append(subnetIDs, *s.SubnetId)
+				}
+			}
+			return subnetIDs
+		}
+		It("should update the VPC config", func() {
+			clusterConfig := makeClusterConfig()
+			ctx := context.Background()
+			awsProvider := makeAWSProvider(ctx, clusterConfig)
+			cluster, err := awsProvider.EKS().DescribeCluster(ctx, &awseks.DescribeClusterInput{
+				Name: aws.String(params.ClusterName),
+			})
+			Expect(err).NotTo(HaveOccurred(), "error describing cluster")
+			clusterSubnetIDs := getPrivateSubnetIDs(ctx, awsProvider.EC2(), *cluster.Cluster.ResourcesVpcConfig.VpcId)
+			Expect(len(cluster.Cluster.ResourcesVpcConfig.SecurityGroupIds) > 0).To(BeTrue(), "at least one security group ID must be associated with the cluster")
+
+			clusterVPC := &api.ClusterVPC{
+				ClusterEndpoints: &api.ClusterEndpoints{
+					PrivateAccess: api.Enabled(),
+					PublicAccess:  api.Enabled(),
+				},
+				PublicAccessCIDRs:            []string{"127.0.0.1/32"},
+				ControlPlaneSubnetIDs:        clusterSubnetIDs,
+				ControlPlaneSecurityGroupIDs: []string{cluster.Cluster.ResourcesVpcConfig.SecurityGroupIds[0]},
+			}
+			By("accepting CLI options")
+			cmd := params.EksctlUtilsCmd.WithArgs(
+				"update-cluster-vpc-config",
+				"--cluster", params.ClusterName,
+				"--private-access",
+				"--public-access",
+				"--public-access-cidrs", strings.Join(clusterVPC.PublicAccessCIDRs, ","),
+				"--control-plane-subnet-ids", strings.Join(clusterVPC.ControlPlaneSubnetIDs, ","),
+				"--control-plane-security-group-ids", strings.Join(clusterVPC.ControlPlaneSecurityGroupIDs, ","),
+				"-v4",
+				"--approve",
+			).
+				WithTimeout(45 * time.Minute)
+			session := cmd.Run()
+			Expect(session.ExitCode()).To(Equal(0))
+
+			formatWithClusterAndRegion := func(format string, values ...any) string {
+				return fmt.Sprintf(format, append([]any{params.ClusterName, params.Region}, values...)...)
+			}
+			Expect(strings.Split(string(session.Buffer().Contents()), "\n")).To(ContainElements(
+				ContainSubstring(formatWithClusterAndRegion("control plane subnets and security groups for cluster %q in %q have been updated to: "+
+					"controlPlaneSubnetIDs=%v, controlPlaneSecurityGroupIDs=%v", clusterVPC.ControlPlaneSubnetIDs, clusterVPC.ControlPlaneSecurityGroupIDs)),
+				ContainSubstring(formatWithClusterAndRegion("Kubernetes API endpoint access for cluster %q in %q has been updated to: privateAccess=%v, publicAccess=%v",
+					*clusterVPC.ClusterEndpoints.PrivateAccess, *clusterVPC.ClusterEndpoints.PublicAccess)),
+				ContainSubstring(formatWithClusterAndRegion("public access CIDRs for cluster %q in %q have been updated to: %v", clusterVPC.PublicAccessCIDRs)),
+			))
+
+			By("accepting a config file")
+			clusterConfig.VPC = clusterVPC
+			cmd = params.EksctlUtilsCmd.WithArgs(
+				"update-cluster-vpc-config",
+				"--config-file", "-",
+				"-v4",
+				"--approve",
+			).
+				WithoutArg("--region", params.Region).
+				WithStdin(clusterutils.Reader(clusterConfig))
+			session = cmd.Run()
+			Expect(session.ExitCode()).To(Equal(0))
+			Expect(strings.Split(string(session.Buffer().Contents()), "\n")).To(ContainElements(
+				ContainSubstring(formatWithClusterAndRegion("Kubernetes API endpoint access for cluster %q in %q is already up-to-date")),
+				ContainSubstring(formatWithClusterAndRegion("control plane subnet IDs for cluster %q in %q are already up-to-date")),
+				ContainSubstring(formatWithClusterAndRegion("control plane security group IDs for cluster %q in %q are already up-to-date")),
+			))
+
+			By("resetting public access CIDRs")
+			cmd = params.EksctlUtilsCmd.WithArgs(
+				"update-cluster-vpc-config",
+				"--cluster", params.ClusterName,
+				"--public-access-cidrs", "0.0.0.0/0",
+				"-v4",
+				"--approve",
+			)
+			Expect(cmd).To(RunSuccessfully())
+		})
+	})
 })
 
 var _ = SynchronizedAfterSuite(func() {}, func() {

pkg/apis/eksctl.io/v1alpha5/assets/schema.json

@@ -655,6 +655,22 @@
           "description": "See [managing access to API](/usage/vpc-networking/#managing-access-to-the-kubernetes-api-server-endpoints)",
           "x-intellij-html-description": "See <a href=\"/usage/vpc-networking/#managing-access-to-the-kubernetes-api-server-endpoints\">managing access to API</a>"
         },
+        "controlPlaneSecurityGroupIDs": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "description": "configures the security groups for the control plane.",
+          "x-intellij-html-description": "configures the security groups for the control plane."
+        },
+        "controlPlaneSubnetIDs": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array",
+          "description": "configures the subnets for the control plane.",
+          "x-intellij-html-description": "configures the subnets for the control plane."
+        },
         "extraCIDRs": {
           "items": {
             "type": "string"
@@ -733,7 +749,9 @@
         "autoAllocateIPv6",
         "nat",
         "clusterEndpoints",
-        "publicAccessCIDRs"
+        "publicAccessCIDRs",
+        "controlPlaneSubnetIDs",
+        "controlPlaneSecurityGroupIDs"
       ],
       "additionalProperties": false,
       "description": "holds global subnet and all child subnets",

pkg/apis/eksctl.io/v1alpha5/validation.go

@@ -318,6 +318,10 @@ func (c *ClusterConfig) ValidateVPCConfig() error {
 		c.VPC.ExtraIPv6CIDRs = cidrs
 	}
 
+	if c.VPC.SecurityGroup != "" && len(c.VPC.ControlPlaneSecurityGroupIDs) > 0 {
+		return errors.New("only one of vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs can be specified")
+	}
+
 	if (c.VPC.IPv6Cidr != "" || c.VPC.IPv6Pool != "") && !c.IPv6Enabled() {
 		return fmt.Errorf("Ipv6Cidr and Ipv6CidrPool are only supported when IPFamily is set to IPv6")
 	}

pkg/apis/eksctl.io/v1alpha5/validation_test.go

@@ -1332,6 +1332,41 @@ var _ = Describe("ClusterConfig validation", func() {
 			})
 
 		})
+
+		type vpcSecurityGroupEntry struct {
+			updateVPC   func(*api.ClusterVPC)
+			expectedErr string
+		}
+		DescribeTable("vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs", func(e vpcSecurityGroupEntry) {
+			e.updateVPC(cfg.VPC)
+			err := cfg.ValidateVPCConfig()
+			if e.expectedErr != "" {
+				Expect(err).To(MatchError(ContainSubstring(e.expectedErr)))
+			} else {
+				Expect(err).NotTo(HaveOccurred())
+			}
+		},
+			Entry("both set", vpcSecurityGroupEntry{
+				updateVPC: func(v *api.ClusterVPC) {
+					v.SecurityGroup = "sg-1234"
+					v.ControlPlaneSecurityGroupIDs = []string{"sg-1234"}
+				},
+				expectedErr: "only one of vpc.securityGroup and vpc.controlPlaneSecurityGroupIDs can be specified",
+			}),
+			Entry("vpc.securityGroup set", vpcSecurityGroupEntry{
+				updateVPC: func(v *api.ClusterVPC) {
+					v.SecurityGroup = "sg-1234"
+				},
+			}),
+			Entry("vpc.controlPlaneSecurityGroupIDs set", vpcSecurityGroupEntry{
+				updateVPC: func(v *api.ClusterVPC) {
+					v.ControlPlaneSecurityGroupIDs = []string{"sg-1234"}
+				},
+			}),
+			Entry("neither set", vpcSecurityGroupEntry{
+				updateVPC: func(v *api.ClusterVPC) {},
+			}),
+		)
 	})
 
 	Describe("ValidatePrivateCluster", func() {

pkg/apis/eksctl.io/v1alpha5/vpc.go

@@ -175,6 +175,12 @@ type (
 		// k8s API endpoint
 		// +optional
 		PublicAccessCIDRs []string `json:"publicAccessCIDRs,omitempty"`
+		// ControlPlaneSubnetIDs configures the subnets for the control plane.
+		// +optional
+		ControlPlaneSubnetIDs []string `json:"controlPlaneSubnetIDs,omitempty"`
+		// ControlPlaneSecurityGroupIDs configures the security groups for the control plane.
+		// +optional
+		ControlPlaneSecurityGroupIDs []string `json:"controlPlaneSecurityGroupIDs,omitempty"`
 	}
 	// ClusterSubnets holds private and public subnets
 	ClusterSubnets struct {

pkg/cfn/builder/cluster.go

@@ -27,7 +27,7 @@ type ClusterResourceSet struct {
 	ec2API         awsapi.EC2
 	region         string
 	vpcResourceSet VPCResourceSet
-	securityGroups []*gfnt.Value
+	securityGroups *gfnt.Value
 }
 
 // NewClusterResourceSet returns a resource set for the new cluster.
@@ -115,7 +115,13 @@ func (c *ClusterResourceSet) AddAllResources(ctx context.Context) error {
 func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *clusterSecurityGroup {
 	var refControlPlaneSG, refClusterSharedNodeSG *gfnt.Value
 
-	if c.spec.VPC.SecurityGroup == "" {
+	if sg := c.spec.VPC.SecurityGroup; sg != "" {
+		refControlPlaneSG = gfnt.NewString(sg)
+		c.securityGroups = gfnt.NewStringSlice(sg)
+	} else if securityGroupIDs := c.spec.VPC.ControlPlaneSecurityGroupIDs; len(securityGroupIDs) > 0 {
+		refControlPlaneSG = gfnt.NewString(securityGroupIDs[0])
+		c.securityGroups = gfnt.NewStringSlice(securityGroupIDs...)
+	} else {
 		refControlPlaneSG = c.newResource(cfnControlPlaneSGResource, &gfnec2.SecurityGroup{
 			GroupDescription: gfnt.NewString("Communication between the control plane and worker nodegroups"),
 			VpcId:            vpcID,
@@ -146,10 +152,8 @@ func (c *ClusterResourceSet) addResourcesForSecurityGroups(vpcID *gfnt.Value) *c
 				})
 			}
 		}
-	} else {
-		refControlPlaneSG = gfnt.NewString(c.spec.VPC.SecurityGroup)
+		c.securityGroups = gfnt.NewSlice(refControlPlaneSG)
 	}
-	c.securityGroups = []*gfnt.Value{refControlPlaneSG} // only this one SG is passed to EKS API, nodes are isolated
 
 	if c.spec.VPC.SharedNodeSecurityGroup == "" {
 		refClusterSharedNodeSG = c.newResource(cfnSharedNodeSGResource, &gfnec2.SecurityGroup{
@@ -263,12 +267,16 @@ func (c *ClusterResourceSet) newResource(name string, resource gfn.Resource) *gf
 
 func (c *ClusterResourceSet) addResourcesForControlPlane(subnetDetails *SubnetDetails) {
 	clusterVPC := &gfneks.Cluster_ResourcesVpcConfig{
-		SubnetIds:             gfnt.NewSlice(subnetDetails.ControlPlaneSubnetRefs()...),
 		EndpointPublicAccess:  gfnt.NewBoolean(*c.spec.VPC.ClusterEndpoints.PublicAccess),
 		EndpointPrivateAccess: gfnt.NewBoolean(*c.spec.VPC.ClusterEndpoints.PrivateAccess),
-		SecurityGroupIds:      gfnt.NewSlice(c.securityGroups...),
+		SecurityGroupIds:      c.securityGroups,
 		PublicAccessCidrs:     gfnt.NewStringSlice(c.spec.VPC.PublicAccessCIDRs...),
 	}
+	if subnetIDs := c.spec.VPC.ControlPlaneSubnetIDs; len(subnetIDs) > 0 {
+		clusterVPC.SubnetIds = gfnt.NewStringSlice(subnetIDs...)
+	} else {
+		clusterVPC.SubnetIds = gfnt.NewSlice(subnetDetails.ControlPlaneSubnetRefs()...)
+	}
 
 	serviceRoleARN := gfnt.MakeFnGetAttString("ServiceRole", "Arn")
 	if api.IsSetAndNonEmptyString(c.spec.IAM.ServiceRoleARN) {