fix: Correct issue where custom launch template is not used when EKS …

…managed node group is used externally (#1824)
baibailiha · Aug 3, 2023 · 926bde2 · 926bde2
1 parent 0cce5e2
commit 926bde2
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 16 deletions.
diff --git a/examples/eks_managed_node_group/README.md b/examples/eks_managed_node_group/README.md
@@ -12,6 +12,33 @@ Configuration in this directory creates an AWS EKS cluster with various EKS Mana
 
 See the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for further details.
 
+## Container Runtime & User Data
+
+When using the default AMI provided by the EKS Managed Node Group service (i.e. - not specifying a value for `ami_id`), users should be aware of the limitations of configuring the node bootstrap process via user data. Due to not having direct access to the bootrap.sh script invocation and therefore its configuration flags (this is provide by the EKS Managed Node Group service in the node user data), a work around for ensuring the appropriate configuration settings is shown below. The following example shows how to inject configuration variables ahead of the merged user data provided by the EKS Managed Node Group service as well as how to enable the containerd runtime using this approach. More details can be found [here](https://github.com/awslabs/amazon-eks-ami/issues/844).
+
+```hcl
+  ...
+  # Demo of containerd usage when not specifying a custom AMI ID
+  # (merged into user data before EKS MNG provided user data)
+  containerd = {
+    name = "containerd"
+
+    # See issue https://github.com/awslabs/amazon-eks-ami/issues/844
+    pre_bootstrap_user_data = <<-EOT
+    #!/bin/bash
+    set -ex
+    cat <<-EOF > /etc/profile.d/bootstrap.sh
+    export CONTAINER_RUNTIME="containerd"
+    export USE_MAX_PODS=false
+    export KUBELET_EXTRA_ARGS="--max-pods=110"
+    EOF
+    # Source extra environment variables in bootstrap script
+    sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
+    EOT
+  }
+  ...
+```
+
 ## Usage
 
 To run this example you need to execute:
@@ -63,6 +90,9 @@ Note that this example may create resources which cost money. Run `terraform des
 | [aws_security_group.remote_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [null_resource.patch](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
+| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_ami.eks_default_arm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
 | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |

diff --git a/examples/eks_managed_node_group/main.tf b/examples/eks_managed_node_group/main.tf
@@ -133,7 +133,7 @@ module "eks" {
     # Custom AMI, using module provided bootstrap data
     bottlerocket_custom = {
       # Current bottlerocket AMI
-      ami_id   = "ami-0ff61e0bcfc81dc94"
+      ami_id   = data.aws_ami.eks_default_bottlerocket.image_id
       platform = "bottlerocket"
 
       # use module user data template to boostrap
@@ -165,7 +165,7 @@ module "eks" {
     custom_ami = {
       ami_type = "AL2_ARM_64"
       # Current default AMI used by managed node groups - pseudo "custom"
-      ami_id = "ami-01dc0aa438e3214c2" # ARM
+      ami_id = data.aws_ami.eks_default_arm.image_id
 
       # This will ensure the boostrap user data is used to join the node
       # By default, EKS managed node groups will not append bootstrap script;
@@ -176,6 +176,25 @@ module "eks" {
       instance_types = ["t4g.medium"]
     }
 
+    # Demo of containerd usage when not specifying a custom AMI ID
+    # (merged into user data before EKS MNG provided user data)
+    containerd = {
+      name = "containerd"
+
+      # See issue https://github.com/awslabs/amazon-eks-ami/issues/844
+      pre_bootstrap_user_data = <<-EOT
+      #!/bin/bash
+      set -ex
+      cat <<-EOF > /etc/profile.d/bootstrap.sh
+      export CONTAINER_RUNTIME="containerd"
+      export USE_MAX_PODS=false
+      export KUBELET_EXTRA_ARGS="--max-pods=110"
+      EOF
+      # Source extra environment variables in bootstrap script
+      sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
+      EOT
+    }
+
     # Complete
     complete = {
       name            = "complete-eks-mng"
@@ -187,23 +206,23 @@ module "eks" {
       max_size     = 7
       desired_size = 1
 
-      ami_id                     = "ami-0caf35bc73450c396"
+      ami_id                     = data.aws_ami.eks_default.image_id
       enable_bootstrap_user_data = true
       bootstrap_extra_args       = "--container-runtime containerd --kubelet-extra-args '--max-pods=20'"
 
       pre_bootstrap_user_data = <<-EOT
-        export CONTAINER_RUNTIME="containerd"
-        export USE_MAX_PODS=false
+      export CONTAINER_RUNTIME="containerd"
+      export USE_MAX_PODS=false
       EOT
 
       post_bootstrap_user_data = <<-EOT
-        echo "you are free little kubelet!"
+      echo "you are free little kubelet!"
       EOT
 
       capacity_type        = "SPOT"
       disk_size            = 256
       force_update_version = true
-      instance_types       = ["m6i.large", "m5.large", "m5n.large", "m5zn.large", "m3.large", "m4.large"]
+      instance_types       = ["m6i.large", "m5.large", "m5n.large", "m5zn.large"]
       labels = {
         GithubRepo = "terraform-aws-eks"
         GithubOrg  = "terraform-aws-modules"
@@ -619,3 +638,33 @@ resource "aws_iam_policy" "node_additional" {
 
   tags = local.tags
 }
+
+data "aws_ami" "eks_default" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amazon-eks-node-${local.cluster_version}-v*"]
+  }
+}
+
+data "aws_ami" "eks_default_arm" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amazon-eks-arm64-node-${local.cluster_version}-v*"]
+  }
+}
+
+data "aws_ami" "eks_default_bottlerocket" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["bottlerocket-aws-k8s-${local.cluster_version}-x86_64-*"]
+  }
+}
diff --git a/examples/self_managed_node_group/README.md b/examples/self_managed_node_group/README.md
@@ -55,7 +55,8 @@ Note that this example may create resources which cost money. Run `terraform des
 | [aws_security_group.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [null_resource.apply](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
 | [tls_private_key.this](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
-| [aws_ami.bottlerocket_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_ami.eks_default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
+| [aws_ami.eks_default_bottlerocket](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_cluster_auth) | data source |
 | [aws_iam_policy_document.ebs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |

diff --git a/examples/self_managed_node_group/main.tf b/examples/self_managed_node_group/main.tf
@@ -94,7 +94,7 @@ module "eks" {
       name = "bottlerocket-self-mng"
 
       platform      = "bottlerocket"
-      ami_id        = data.aws_ami.bottlerocket_ami.id
+      ami_id        = data.aws_ami.eks_default_bottlerocket.id
       instance_type = "m5.large"
       desired_size  = 2
       key_name      = aws_key_pair.this.key_name
@@ -159,16 +159,16 @@ module "eks" {
       max_size     = 7
       desired_size = 1
 
-      ami_id               = "ami-0caf35bc73450c396"
+      ami_id               = data.aws_ami.eks_default.id
       bootstrap_extra_args = "--kubelet-extra-args '--max-pods=110'"
 
       pre_bootstrap_user_data = <<-EOT
-        export CONTAINER_RUNTIME="containerd"
-        export USE_MAX_PODS=false
+      export CONTAINER_RUNTIME="containerd"
+      export USE_MAX_PODS=false
       EOT
 
       post_bootstrap_user_data = <<-EOT
-        echo "you are free little kubelet!"
+      echo "you are free little kubelet!"
       EOT
 
       disk_size     = 256
@@ -374,7 +374,17 @@ resource "aws_kms_key" "eks" {
   tags = local.tags
 }
 
-data "aws_ami" "bottlerocket_ami" {
+data "aws_ami" "eks_default" {
+  most_recent = true
+  owners      = ["amazon"]
+
+  filter {
+    name   = "name"
+    values = ["amazon-eks-node-${local.cluster_version}-v*"]
+  }
+}
+
+data "aws_ami" "eks_default_bottlerocket" {
   most_recent = true
   owners      = ["amazon"]
 

diff --git a/modules/eks-managed-node-group/main.tf b/modules/eks-managed-node-group/main.tf
@@ -30,8 +30,14 @@ module "user_data" {
 ################################################################################
 
 locals {
-  use_custom_launch_template = var.launch_template_name != ""
-  launch_template_name_int   = coalesce(var.launch_template_name, "${var.name}-eks-node-group")
+  # There are 4 scenarios here that have to be considered for `use_custom_launch_template`:
+  # 1. `var.create_launch_template = false && var.launch_template_name == ""` => EKS MNG will use its own default LT
+  # 2. `var.create_launch_template = false && var.launch_template_name == "something"` => User provided custom LT will be used
+  # 3. `var.create_launch_template = true && var.launch_template_name == ""` => Custom LT will be used, module will provide a default name
+  # 4. `var.create_launch_template = true && var.launch_template_name == "something"` => Custom LT will be used, LT name is provided by user
+  use_custom_launch_template = var.create_launch_template || var.launch_template_name != ""
+
+  launch_template_name_int = coalesce(var.launch_template_name, "${var.name}-eks-node-group")
 }
 
 resource "aws_launch_template" "this" {