[CARMEL-3499] Change vCores to 1 in ContainerRequest for hadoop limit (…

…#17) * [CARMEL-3499] Change vCores to 1 in ContainerRequest for hadoop limit * change the default value of DEFAULT_RESOURCE_CALCULATOR to false
apache · Aug 11, 2020 · e933ef6 · e933ef6
1 parent 171868b
commit e933ef6
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 0 deletions.
diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -406,6 +406,9 @@ private[yarn] class YarnAllocator(
       resource: Resource,
       nodes: Array[String],
       racks: Array[String]): ContainerRequest = {
+    if (sparkConf.get(DEFAULT_RESOURCE_CALCULATOR)) {
+      resource.setVirtualCores(1)
+    }
     new ContainerRequest(resource, nodes, racks, RM_REQUEST_PRIORITY, true, labelExpression.orNull)
   }
 

diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala
@@ -178,6 +178,13 @@ package object config {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("1s")
 
+  private[spark] val DEFAULT_RESOURCE_CALCULATOR =
+    ConfigBuilder("spark.yarn.scheduler.default.resource.calculator")
+      .doc("After enable default resource calculator, the value of vCores " +
+        "in ContainerRequest is always 1.")
+      .booleanConf
+      .createWithDefault(false)
+
   /* Shared Client-mode AM / Driver configuration. */
 
   private[spark] val AM_MAX_WAIT_TIME = ConfigBuilder("spark.yarn.am.waitTime")