[CI] Verify kubectl in kind-in-docker step (#1305)

This PR configures kind to allow kubectl to connect to it. This is a prerequisite for testing sample YAML files end-to-end in Buildkite. The configuration is copied from @simon-mo's work in ray-project/ray#22035. --------- Signed-off-by: Archit Kulkarni <[email protected]>
ray-project · Aug 9, 2023 · 9e37e19 · 9e37e19
1 parent 7386427
commit 9e37e19
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 4 deletions.
diff --git a/.buildkite/kind-in-docker.yml b/.buildkite/kind-in-docker.yml
@@ -1,4 +1,4 @@
-- label: kind in Docker
+- label: kind and kubectl
   instance_size: large
   image: golang:1.17
   commands:
@@ -13,6 +13,24 @@
     # Install Docker
     - bash scripts/install-docker.sh
 
-    # Delete cluster if it already exists
-    - kind delete cluster
-    - kind create cluster
+    # Delete dangling clusters
+    - kind delete clusters --all
+
+    # Install kubectl.
+    - curl -LO https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl
+    - curl -LO "https://dl.k8s.io/release/v1.27.3/bin/linux/amd64/kubectl.sha256"
+    - echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check
+    - install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+    # Create the cluster
+    - time kind create cluster --wait 120s --config tests/framework/config/kind-config-buildkite.yml
+    - docker ps
+
+    # Now the kind node is running, it exposes port 6443 in the dind-daemon network.
+    - kubectl config set clusters.kind-kind.server https://docker:6443
+
+    # Verify that kubectl works
+    - kubectl version
+    - kubectl cluster-info
+    - kubectl get nodes
+    - kubectl get pods --all-namespaces
diff --git a/tests/framework/config/kind-config-buildkite.yml b/tests/framework/config/kind-config-buildkite.yml
@@ -0,0 +1,17 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerAddress: "0.0.0.0"
+  # Ensure stable port so we can rewrite the server address later
+  apiServerPort: 6443
+
+# Adding this so containers from the same docker network can access it
+# https://blog.scottlowe.org/2019/07/30/adding-a-name-to-kubernetes-api-server-certificate/
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    apiServer:
+      certSANs:
+        - "docker"