Add a flag to enable/disable worker init container injection

helm-chart/kuberay-operator/values.yaml

@@ -67,6 +67,9 @@ singleNamespaceInstall: false
 # watchNamespace: ray-user-namespace
 
 # Environment variables
-env: []
-# - name: EXAMPLE_ENV
-#   value: "1"
+env:
+# If not set or set to true, kuberay auto injects an init container waiting for ray GCS.
+# If false, you will need to inject your own init container to ensure ray GCS is up before the ray workers start.
+# Warning: we highly recommend setting to true and let kuberay handle for you.
+# - name: ENABLE_INIT_CONTAINER_INJECTION
+#   value: "true"

ray-operator/controllers/ray/common/pod.go

@@ -3,6 +3,7 @@ package common
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
@@ -28,6 +29,9 @@ const (
 	ObjectStoreMemoryKey        = "object-store-memory"
 	// TODO (davidxia): should be a const in upstream ray-project/ray
 	AllowSlowStorageEnvVar = "RAY_OBJECT_STORE_ALLOW_SLOW_STORAGE"
+	// If set to true, kuberay auto injects an init container waiting for ray GCS.
+	// If false, you will need to inject your own init container to ensure ray GCS is up before the ray workers start.
+	EnableInitContainerInjectionEnvKey = "ENABLE_INIT_CONTAINER_INJECTION"
 )
 
 var log = logf.Log.WithName("RayCluster-Controller")
@@ -180,6 +184,13 @@ func autoscalerSupportIsStable(rayVersion string) bool {
 	}
 }
 
+func getEnableInitContainerInjection() bool {
+	if s := os.Getenv(EnableInitContainerInjectionEnvKey); strings.ToLower(s) == "false" {
+		return false
+	}
+	return true
+}
+
 // DefaultWorkerPodTemplate sets the config values
 func DefaultWorkerPodTemplate(instance rayiov1alpha1.RayCluster, workerSpec rayiov1alpha1.WorkerGroupSpec, podName string, fqdnRayIP string, headPort string) v1.PodTemplateSpec {
 	podTemplate := workerSpec.Template
@@ -191,27 +202,32 @@ func DefaultWorkerPodTemplate(instance rayiov1alpha1.RayCluster, workerSpec rayi
 
 	// The Ray worker should only start once the GCS server is ready.
 	rayContainerIndex := getRayContainerIndex(podTemplate.Spec)
-	// Do not modify `deepCopyRayContainer` anywhere.
-	deepCopyRayContainer := podTemplate.Spec.Containers[rayContainerIndex].DeepCopy()
-	initContainer := v1.Container{
-		Name:            "wait-gcs-ready",
-		Image:           podTemplate.Spec.Containers[rayContainerIndex].Image,
-		ImagePullPolicy: v1.PullIfNotPresent,
-		Command:         []string{"/bin/bash", "-lc", "--"},
-		Args: []string{
-			fmt.Sprintf("until ray health-check --address %s:%s > /dev/null 2>&1; do echo wait for GCS to be ready; sleep 5; done", fqdnRayIP, headPort),
-		},
-		SecurityContext: podTemplate.Spec.Containers[rayContainerIndex].SecurityContext.DeepCopy(),
-		// This init container requires certain environment variables to establish a secure connection with the Ray head using TLS authentication.
-		// Additionally, some of these environment variables may reference files stored in volumes, so we need to include both the `Env` and `VolumeMounts` fields here.
-		// For more details, please refer to: https://docs.ray.io/en/latest/ray-core/configure.html#tls-authentication.
-		Env:          deepCopyRayContainer.Env,
-		VolumeMounts: deepCopyRayContainer.VolumeMounts,
-		// If users specify ResourceQuota for the namespace, the init container need to specify resource explicitly.
-		Resources: deepCopyRayContainer.Resources,
-	}
-	podTemplate.Spec.InitContainers = append(podTemplate.Spec.InitContainers, initContainer)
 
+	// only inject init container only when ENABLE_INIT_CONTAINER_INJECTION is true
+	enableInitContainerInjection := getEnableInitContainerInjection()
+
+	if enableInitContainerInjection {
+		// Do not modify `deepCopyRayContainer` anywhere.
+		deepCopyRayContainer := podTemplate.Spec.Containers[rayContainerIndex].DeepCopy()
+		initContainer := v1.Container{
+			Name:            "wait-gcs-ready",
+			Image:           podTemplate.Spec.Containers[rayContainerIndex].Image,
+			ImagePullPolicy: v1.PullIfNotPresent,
+			Command:         []string{"/bin/bash", "-lc", "--"},
+			Args: []string{
+				fmt.Sprintf("until ray health-check --address %s:%s > /dev/null 2>&1; do echo wait for GCS to be ready; sleep 5; done", fqdnRayIP, headPort),
+			},
+			SecurityContext: podTemplate.Spec.Containers[rayContainerIndex].SecurityContext.DeepCopy(),
+			// This init container requires certain environment variables to establish a secure connection with the Ray head using TLS authentication.
+			// Additionally, some of these environment variables may reference files stored in volumes, so we need to include both the `Env` and `VolumeMounts` fields here.
+			// For more details, please refer to: https://docs.ray.io/en/latest/ray-core/configure.html#tls-authentication.
+			Env:          deepCopyRayContainer.Env,
+			VolumeMounts: deepCopyRayContainer.VolumeMounts,
+			// If users specify ResourceQuota for the namespace, the init container need to specify resource explicitly.
+			Resources: deepCopyRayContainer.Resources,
+		}
+		podTemplate.Spec.InitContainers = append(podTemplate.Spec.InitContainers, initContainer)
+	}
 	// If the replica of workers is more than 1, `ObjectMeta.Name` may cause name conflict errors.
 	// Hence, we set `ObjectMeta.Name` to an empty string, and use GenerateName to prevent name conflicts.
 	podTemplate.ObjectMeta.Name = ""

ray-operator/controllers/ray/common/pod_test.go

@@ -2,6 +2,7 @@ package common
 
 import (
 	"fmt"
+	"os"
 	"reflect"
 	"sort"
 	"strings"
@@ -979,3 +980,28 @@ func TestSetMissingRayStartParamsBlock(t *testing.T) {
 	rayStartParams = setMissingRayStartParams(rayStartParams, rayiov1alpha1.WorkerNode, headPort, fqdnRayIP)
 	assert.Equal(t, "false", rayStartParams["block"], fmt.Sprintf("Expected `%v` but got `%v`", "false", rayStartParams["block"]))
 }
+
+func TestGetCustomWorkerInitImage(t *testing.T) {
+	// cleanup
+	defer os.Unsetenv(EnableInitContainerInjectionEnvKey)
+
+	// not set the env
+	b := getEnableInitContainerInjection()
+	assert.True(t, b)
+	// set the env with "true"
+	os.Setenv(EnableInitContainerInjectionEnvKey, "true")
+	b = getEnableInitContainerInjection()
+	assert.True(t, b)
+	// set the env with "True"
+	os.Setenv(EnableInitContainerInjectionEnvKey, "True")
+	b = getEnableInitContainerInjection()
+	assert.True(t, b)
+	// set the env with "false"
+	os.Setenv(EnableInitContainerInjectionEnvKey, "false")
+	b = getEnableInitContainerInjection()
+	assert.False(t, b)
+	// set the env with "False"
+	os.Setenv(EnableInitContainerInjectionEnvKey, "False")
+	b = getEnableInitContainerInjection()
+	assert.False(t, b)
+}