feat: Add example for Karpenter GPU autoscaling

karpenter-gpu/README.md

@@ -0,0 +1,70 @@
+# Karpenter GPU Autoscaling
+
+### Prerequisites:
+
+Ensure that you have the following tools installed locally:
+
+1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)
+2. [kubectl](https://Kubernetes.io/docs/tasks/tools/)
+3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli)
+
+### Deployment
+
+1. Provision resources as they are defined in the directory with the following. Note - ensure your CLI session is authenticated with AWS to provision resources.
+
+```bash
+terraform init -upgrade=true
+terraform apply -target=module.vpc -target=module.eks
+terraform apply -target='module.eks_blueprints_addons.module.karpenter' -target=kubectl_manifest.karpenter_node_template
+```
+
+2. Once the cluster is up and running and the node group is provisioned, update your Terraform state to align with changes made by the AWS API. This doesn't modify any resources, it just simply aligns your statefile with the current state. You can read more about this at the following links if interested:
+
+- https://github.com/hashicorp/terraform/pull/28634
+- https://github.com/hashicorp/terraform/issues/28803
+
+```bash
+terraform apply -refresh-only
+terraform plan # should show `No changes. Your infrastructure matches the configuration.`
+```
+
+3. With the cluster up and running we can check that Weaviate is functioning as intended. First, update your kubeconfig to access the cluster:
+
+```bash
+# First, make sure you have updated your local kubeconfig
+aws eks --region us-west-2 update-kubeconfig --name karpenter-gpu
+```
+
+4. Deploy the sample GPU deployment:
+
+```bash
+kubectl apply -f gpu.yaml
+```
+
+5. Scale up the deployment and verify the pods deploy onto GPU (p or g-class) nodes:
+
+```bash
+kubectl scale deployment test-gpu --replicas=4
+
+# Output should look similar to below
+NAME                        READY   STATUS    RESTARTS   AGE
+test-gpu-5964c55c7b-57zbd   1/1     Running   0          4m58s
+test-gpu-5964c55c7b-gbw6r   1/1     Running   0          4m58s
+test-gpu-5964c55c7b-lmkzh   1/1     Running   0          4m58s
+test-gpu-5964c55c7b-qvfmp   1/1     Running   0          4m58s
+```
+
+### Tear Down & Clean-Up
+
+1. Scale down the deployment so that the Karpenter created nodes are removed:
+
+```bash
+kubectl delete -f gpu.yaml
+```
+
+2. Remove the resources created by Terraform
+
+```bash
+terraform destroy -target=module.eks_blueprints_addons
+terraform destroy
+```

karpenter-gpu/add-ons.tf

@@ -0,0 +1,156 @@
+################################################################################
+# Addons
+################################################################################
+
+data "aws_ecrpublic_authorization_token" "this" {
+  provider = aws.useast1
+}
+
+module "eks_blueprints_addons" {
+  source  = "aws-ia/eks-blueprints-addons/aws"
+  version = "~> 1.0"
+
+  cluster_name      = module.eks.cluster_name
+  cluster_endpoint  = module.eks.cluster_endpoint
+  cluster_version   = module.eks.cluster_version
+  oidc_provider_arn = module.eks.oidc_provider_arn
+
+  # Wait for compute to be available
+  create_delay_dependencies = [for group in module.eks.eks_managed_node_groups :
+    group.node_group_arn if group.node_group_arn != null
+  ]
+
+  enable_karpenter                  = true
+  karpenter_enable_spot_termination = true
+  karpenter = {
+    repository_username = data.aws_ecrpublic_authorization_token.this.user_name
+    repository_password = data.aws_ecrpublic_authorization_token.this.password
+  }
+
+  tags = module.tags.tags
+}
+
+################################################################################
+# Default - Karpenter Provisioner
+################################################################################
+
+resource "kubectl_manifest" "karpenter_provisioner" {
+  yaml_body = <<-YAML
+    apiVersion: karpenter.sh/v1alpha5
+    kind: Provisioner
+    metadata:
+      name: default
+    spec:
+      providerRef:
+        name: default
+      requirements:
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["on-demand"]
+      ttlSecondsAfterEmpty: 30
+  YAML
+
+  depends_on = [
+    module.eks_blueprints_addons.karpenter
+  ]
+}
+
+resource "kubectl_manifest" "karpenter_node_template" {
+  yaml_body = <<-YAML
+    apiVersion: karpenter.k8s.aws/v1alpha1
+    kind: AWSNodeTemplate
+    metadata:
+      name: default
+    spec:
+      amiFamily: Bottlerocket
+      metadataOptions:
+        httpEndpoint: enabled
+        httpPutResponseHopLimit: 2
+        httpTokens: required
+      subnetSelector:
+        karpenter.sh/discovery: ${module.eks.cluster_name}
+      securityGroupSelector:
+        karpenter.sh/discovery: ${module.eks.cluster_name}
+      tags:
+        karpenter.sh/discovery: ${module.eks.cluster_name}
+  YAML
+
+  depends_on = [
+    kubectl_manifest.karpenter_provisioner
+  ]
+}
+
+################################################################################
+# GPU - Karpenter Provisioner
+################################################################################
+
+resource "kubectl_manifest" "karpenter_provisioner_gpu" {
+  yaml_body = <<-YAML
+    apiVersion: karpenter.sh/v1alpha5
+    kind: Provisioner
+    metadata:
+      name: gpu
+    spec:
+      providerRef:
+        name: gpu
+      requirements:
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["on-demand"]
+        - key: "karpenter.k8s.aws/instance-cpu"
+          operator: Gt
+          values: ["6"]
+        - key: "karpenter.k8s.aws/instance-category"
+          operator: In
+          values: ["g", "p"]
+        - key: "karpenter.k8s.aws/instance-generation"
+          operator: Gt
+          values: ["2"]
+      taints:
+        - key: nvidia.com/gpu
+          effect: "NoSchedule"
+      ttlSecondsAfterEmpty: 30
+  YAML
+
+  depends_on = [
+    module.eks_blueprints_addons.karpenter
+  ]
+}
+
+resource "kubectl_manifest" "karpenter_node_template_gpu" {
+  yaml_body = <<-YAML
+    apiVersion: karpenter.k8s.aws/v1alpha1
+    kind: AWSNodeTemplate
+    metadata:
+      name: gpu
+    spec:
+      amiFamily: Bottlerocket
+      metadataOptions:
+        httpEndpoint: enabled
+        httpPutResponseHopLimit: 2
+        httpTokens: required
+      blockDeviceMappings:
+        # Root device
+        - deviceName: /dev/xvda
+          ebs:
+            volumeSize: 10Gi
+            volumeType: gp3
+            encrypted: true
+        # Data device: Container resources such as images and logs
+        - deviceName: /dev/xvdb
+          ebs:
+            volumeSize: 64Gi
+            volumeType: gp3
+            encrypted: true
+      subnetSelector:
+        karpenter.sh/discovery: ${module.eks.cluster_name}
+      securityGroupSelector:
+        karpenter.sh/discovery: ${module.eks.cluster_name}
+      tags:
+        karpenter.sh/discovery: ${module.eks.cluster_name}
+  YAML
+
+  depends_on = [
+    kubectl_manifest.karpenter_provisioner_gpu
+  ]
+}

karpenter-gpu/eks.tf

@@ -0,0 +1,73 @@
+################################################################################
+# EKS Cluster
+################################################################################
+
+module "eks" {
+  source  = "terraform-aws-modules/eks/aws"
+  version = "~> 19.15"
+
+  cluster_name    = local.name
+  cluster_version = "1.27"
+
+  cluster_endpoint_public_access = true
+
+  cluster_addons = {
+    aws-ebs-csi-driver = {
+      service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn
+    }
+    coredns    = {}
+    kube-proxy = {}
+    vpc-cni    = {}
+  }
+
+  vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
+
+  manage_aws_auth_configmap = true
+  aws_auth_roles = [
+    # Karpenter node IAM role for nodes launched by Karpenter
+    {
+      rolearn  = module.eks_blueprints_addons.karpenter.node_iam_role_arn
+      username = "system:node:{{EC2PrivateDNSName}}"
+      groups = [
+        "system:bootstrappers",
+        "system:nodes",
+      ]
+    },
+  ]
+
+  eks_managed_node_groups = {
+    default = {
+      instance_types = ["m5.large"]
+
+      min_size     = 2
+      max_size     = 3
+      desired_size = 2
+    }
+  }
+
+  tags = merge(module.tags.tags, {
+    # NOTE - if creating multiple security groups with this module, only tag the
+    # security group that Karpenter should utilize with the following tag
+    # (i.e. - at most, only one security group should have this tag in your account)
+    "karpenter.sh/discovery" = local.name
+  })
+}
+
+module "ebs_csi_driver_irsa" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version = "~> 5.20"
+
+  role_name_prefix = "${module.eks.cluster_name}-ebs-csi-driver-"
+
+  attach_ebs_csi_policy = true
+
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
+    }
+  }
+
+  tags = module.tags.tags
+}

karpenter-gpu/gpu.yaml

@@ -0,0 +1,28 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: test-gpu
+  labels:
+    app: gpu
+spec:
+  replicas: 0
+  selector:
+    matchLabels:
+      app: gpu
+  template:
+    metadata:
+      labels:
+        app: gpu
+    spec:
+      tolerations:
+      - key: 'nvidia.com/gpu'
+        operator: 'Exists'
+        effect: 'NoSchedule'
+      containers:
+      - name: gpu-test
+        image: "oguzpastirmaci/gpu-burn"
+        imagePullPolicy: IfNotPresent
+        command: ["bash", "-c", "while true; do /app/gpu_burn 20; sleep 20; done"]
+        resources:
+          limits:
+            nvidia.com/gpu: "1"