[CI] Run sample job YAML tests in buildkite (ray-project#1315)

Sets up the Buildkite CI pipeline to test the RayJob sample YAML files using kind. Related issue number Closes ray-project#1246 --------- Signed-off-by: Archit Kulkarni <[email protected]>
blublinsky · Aug 15, 2023 · c3a0285 · c3a0285
1 parent 44c0cec
commit c3a0285
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 8 deletions.
diff --git a/.buildkite/kind-in-docker.yml → .buildkite/test-rayjob-sample-yamls.yml b/.buildkite/kind-in-docker.yml → .buildkite/test-rayjob-sample-yamls.yml
@@ -1,4 +1,4 @@
-- label: KubeRay operator on kind
+- label: Test RayJob Sample YAMLs
   instance_size: large
   image: golang:1.17
   commands:
@@ -54,3 +54,16 @@
     - popd
 
     - echo "Kuberay operator successfully installed."
+
+    # Delete kind clusters
+    - kind delete clusters --all
+
+    # Install python 3.10 and pip
+    - apt-get update
+    - apt-get install -y python3.10 python3-pip
+
+    # Install requirements
+    - pip install -r tests/framework/config/requirements.txt
+
+    # Run test
+    - BUILDKITE_ENV=true python3 tests/test_sample_rayjob_yamls.py
diff --git a/tests/framework/config/kind-config-buildkite.yml b/tests/framework/config/kind-config-buildkite.yml
@@ -14,4 +14,26 @@ nodes:
     kind: ClusterConfiguration
     apiServer:
       certSANs:
-        - "docker"
+        - "docker"
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+    - containerPort: 30265
+      hostPort: 8265
+      listenAddress: "0.0.0.0"
+      protocol: tcp
+    - containerPort: 30001
+      hostPort: 10001
+      listenAddress: "0.0.0.0"
+      protocol: tcp
+    - containerPort: 32365
+      hostPort: 52365
+      listenAddress: "0.0.0.0"
+      protocol: tcp
+    - containerPort: 30800
+      hostPort: 8000
+      listenAddress: "0.0.0.0"
+      protocol: tcp
diff --git a/tests/framework/config/kind-config.yaml b/tests/framework/config/kind-config.yaml
@@ -9,10 +9,6 @@ nodes:
           kubeletExtraArgs:
             node-labels: "ingress-ready=true"
     extraPortMappings:
-      - containerPort: 30379
-        hostPort: 6379
-        listenAddress: "0.0.0.0"
-        protocol: tcp
       - containerPort: 30265
         hostPort: 8265
         listenAddress: "0.0.0.0"

diff --git a/tests/framework/utils.py b/tests/framework/utils.py
@@ -1,5 +1,6 @@
 """Utilities for integration tests of KubeRay."""
 
+import os
 import subprocess
 import logging
 from pathlib import Path
@@ -34,7 +35,13 @@ class CONST:
     # Paths
     REPO_ROOT = Path(__file__).absolute().parent.parent.parent
     HELM_CHART_ROOT = REPO_ROOT.joinpath("helm-chart")
-    DEFAULT_KIND_CONFIG = REPO_ROOT.joinpath("tests/framework/config/kind-config.yaml")
+
+    # Decide the config based on the environment
+    BUILDKITE_ENV = "BUILDKITE_ENV"
+    if os.getenv(BUILDKITE_ENV, default="") == "true":
+        DEFAULT_KIND_CONFIG = REPO_ROOT.joinpath("tests/framework/config/kind-config-buildkite.yml")
+    else:
+        DEFAULT_KIND_CONFIG = REPO_ROOT.joinpath("tests/framework/config/kind-config.yaml")
 
     # Ray features
     RAY_FT = "RAY_FT"
@@ -52,7 +59,7 @@ class CONST:
 
 class KubernetesClusterManager:
     """
-    KubernetesClusterManager controlls the lifecycle of KinD cluster and Kubernetes API client.
+    KubernetesClusterManager controls the lifecycle of KinD cluster and Kubernetes API client.
     """
 
     def __init__(self) -> None:
@@ -66,11 +73,20 @@ def delete_kind_cluster(self) -> None:
             k8s_client.api_client.close()
         self.k8s_client_dict = {}
 
+    def _adjust_kubeconfig_server_address(self) -> None:
+        """Modify the server address in kubeconfig to https://docker:6443"""
+        if os.getenv(CONST.BUILDKITE_ENV, default="") == "true":
+            shell_subprocess_run("kubectl config set clusters.kind-kind.server https://docker:6443")
+
     def create_kind_cluster(self, kind_config=None) -> None:
         """Create a KinD cluster"""
         # To use NodePort service, `kind_config` needs to set `extraPortMappings` properly.
         kind_config = CONST.DEFAULT_KIND_CONFIG if not kind_config else kind_config
         shell_subprocess_run(f"kind create cluster --wait 900s --config {kind_config}")
+
+        # Adjust the kubeconfig server address if necessary
+        self._adjust_kubeconfig_server_address()
+
         config.load_kube_config()
         self.k8s_client_dict.update(
             {