From e933ef6b02ac42718fdfde873ec0aa95d2c5cd7a Mon Sep 17 00:00:00 2001 From: "Luan, Xuedong" Date: Tue, 11 Aug 2020 16:58:45 +0800 Subject: [PATCH] [CARMEL-3499] Change vCores to 1 in ContainerRequest for hadoop limit (#17) * [CARMEL-3499] Change vCores to 1 in ContainerRequest for hadoop limit * change the default value of DEFAULT_RESOURCE_CALCULATOR to false --- .../scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 3 +++ .../main/scala/org/apache/spark/deploy/yarn/config.scala | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala index 09414cbbe50a4..845dac6199099 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala @@ -406,6 +406,9 @@ private[yarn] class YarnAllocator( resource: Resource, nodes: Array[String], racks: Array[String]): ContainerRequest = { + if (sparkConf.get(DEFAULT_RESOURCE_CALCULATOR)) { + resource.setVirtualCores(1) + } new ContainerRequest(resource, nodes, racks, RM_REQUEST_PRIORITY, true, labelExpression.orNull) } diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala index 3797491bb2e92..6ff51f96591e5 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/config.scala @@ -178,6 +178,13 @@ package object config { .timeConf(TimeUnit.MILLISECONDS) .createWithDefaultString("1s") + private[spark] val DEFAULT_RESOURCE_CALCULATOR = + ConfigBuilder("spark.yarn.scheduler.default.resource.calculator") + .doc("After enable default resource calculator, the value of vCores " + + "in ContainerRequest is always 1.") + .booleanConf + .createWithDefault(false) + /* Shared Client-mode AM / Driver configuration. */ private[spark] val AM_MAX_WAIT_TIME = ConfigBuilder("spark.yarn.am.waitTime")