[CI] Add K8s Builder Step (#22035)

.buildkite/pipeline.yml

@@ -416,3 +416,8 @@
       - ./ci/travis/env_info.sh
       - python ./ci/travis/check_minimal_install.py
       - bazel test --config=ci $(./scripts/bazel_export_options)  --build_tests_only --test_tag_filters=minimal python/ray/train/...
+
+- label: ":kubernetes: operator"
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/travis/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/travis/prep-k8s-environment.sh

ci/travis/kind.config.yaml

@@ -0,0 +1,17 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  apiServerAddress: "0.0.0.0"
+  # Ensure stable port so we can rewrite the server address later
+  apiServerPort: 6443
+
+# Adding this so containers from the same docker network can access it
+# https://blog.scottlowe.org/2019/07/30/adding-a-name-to-kubernetes-api-server-certificate/
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    apiServer:
+      certSANs:
+        - "docker"

ci/travis/prep-k8s-environment.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# This scripts creates a kind cluster and verify it works
+
+set -xe
+
+# Install kind
+wget https://github.com/kubernetes-sigs/kind/releases/download/v0.11.1/kind-linux-amd64
+chmod +x kind-linux-amd64
+mv ./kind-linux-amd64 /usr/bin/kind
+kind --help
+
+# Install kubectl
+curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+chmod +x kubectl
+mv ./kubectl /usr/bin/kubectl
+kubectl version --client
+
+# Create the cluster
+time kind create cluster --wait 120s --config ./ci/travis/kind.config.yaml
+docker ps
+
+# Now the kind node is running, it exposes port 6443 in the dind-daemon network.
+kubectl config set clusters.kind-kind.server https://docker:6443
+
+# Verify the kubectl works
+kubectl version
+kubectl cluster-info
+kubectl get nodes
+kubectl get pods --all-namespaces
+