dynamic allocation executor pending for addition flood fix (#396)

* dynamicallocation executor pending for addition flood fix * scala checkstyle fix * scala checkstyle fix * scala checkstyle fix
oap-project · Jun 26, 2024 · 8739d52 · 8739d52
1 parent cbce5df
commit 8739d52
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayAppMaster.scala b/core/raydp-main/src/main/scala/org/apache/spark/deploy/raydp/RayAppMaster.scala
@@ -270,6 +270,16 @@ class RayAppMaster(host: String,
           .map{ case (name, amount) => s"${name}: ${amount}"}.mkString(", ")} }..")
       // TODO: Support generic fractional logical resources using prefix spark.ray.actor.resource.*
 
+      // This will check with dynamic auto scale no additional pending executor actor added more
+      // than max executors count as this result in executor even running after job completion
+      val dynamicAllocationEnabled = conf.getBoolean("spark.dynamicAllocation.enabled", false)
+      if (dynamicAllocationEnabled) {
+        val maxExecutor = conf.getInt("spark.dynamicAllocation.maxExecutors", 0)
+        if (restartedExecutors.size >= maxExecutor) {
+          return
+        }
+      }
+
       val handler = RayExecutorUtils.createExecutorActor(
         executorId, getAppMasterEndpointUrl(),
         rayActorCPU,