From f285081c8307e090638213b4e26cf5c35072b0e0 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 30 Aug 2018 14:05:43 +0100 Subject: [PATCH 01/10] read hadoop config dir from spark property --- .../org/apache/spark/deploy/SparkSubmit.scala | 1 + .../spark/deploy/SparkSubmitArguments.scala | 6 ++-- .../launcher/AbstractCommandBuilder.java | 1 + .../org/apache/spark/deploy/yarn/Client.scala | 28 ++++++++++--------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index cf902db8709e7..cd0e748715129 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -530,6 +530,7 @@ private[spark] class SparkSubmit extends Logging { CLUSTER, confKey = "spark.jars.excludes"), // Yarn only + OptionAssigner(args.) OptionAssigner(args.queue, YARN, ALL_DEPLOY_MODES, confKey = "spark.yarn.queue"), OptionAssigner(args.numExecutors, YARN, ALL_DEPLOY_MODES, confKey = "spark.executor.instances"), diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 0998757715457..842460821ed22 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -286,9 +286,11 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S if (master.startsWith("yarn")) { val hasHadoopEnv = env.contains("HADOOP_CONF_DIR") || env.contains("YARN_CONF_DIR") - if (!hasHadoopEnv && !Utils.isTesting) { + val hasHadoopProp = sparkProperties.contains("spark.yarn.config.dir") + if (!hasHadoopEnv && !hasHadoopProp && !Utils.isTesting) { error(s"When running with master '$master' " + - "either HADOOP_CONF_DIR or YARN_CONF_DIR must be set in the environment.") + "either HADOOP_CONF_DIR or YARN_CONF_DIR must be set in the environment, +" + + "or spark.yarn.config.dir in the spark properties.") } } diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index ce24400f557cd..9bde2192f8708 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -199,6 +199,7 @@ List buildClassPath(String appClassPath) throws IOException { } addToClassPath(cp, getenv("HADOOP_CONF_DIR")); + addToClassPath(cp, getEffectiveConfig().get("spark.yarn.config.dir")); addToClassPath(cp, getenv("YARN_CONF_DIR")); addToClassPath(cp, getenv("SPARK_DIST_CLASSPATH")); return new ArrayList<>(cp); diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 75614a41e0b62..6afd13ea51045 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -686,24 +686,26 @@ private[spark] class Client( } } + val configDirProperty = sparkConf.getOption("spark.hadoop.config.dir") + // SPARK-23630: during testing, Spark scripts filter out hadoop conf dirs so that user's // environments do not interfere with tests. This allows a special env variable during // tests so that custom conf dirs can be used by unit tests. - val confDirs = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") ++ + val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") ++ (if (Utils.isTesting) Seq("SPARK_TEST_HADOOP_CONF_DIR") else Nil) - confDirs.foreach { envKey => - sys.env.get(envKey).foreach { path => - val dir = new File(path) - if (dir.isDirectory()) { - val files = dir.listFiles() - if (files == null) { - logWarning("Failed to list files under directory " + dir) - } else { - files.foreach { file => - if (file.isFile && !hadoopConfFiles.contains(file.getName())) { - hadoopConfFiles(file.getName()) = file - } + val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProperty).flatMap(_.toList) + confDirPaths.foreach { path => + logDebug("Reading config files from " + path) + val dir = new File(path) + if (dir.isDirectory()) { + val files = dir.listFiles() + if (files == null) { + logWarning("Failed to list files under directory " + dir) + } else { + files.foreach { file => + if (file.isFile && !hadoopConfFiles.contains(file.getName())) { + hadoopConfFiles(file.getName()) = file } } } From 503c9859c37afbf753ee70459c122199d4942af2 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 30 Aug 2018 15:12:47 +0100 Subject: [PATCH 02/10] renames --- .../src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | 1 - .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala index cd0e748715129..cf902db8709e7 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala @@ -530,7 +530,6 @@ private[spark] class SparkSubmit extends Logging { CLUSTER, confKey = "spark.jars.excludes"), // Yarn only - OptionAssigner(args.) OptionAssigner(args.queue, YARN, ALL_DEPLOY_MODES, confKey = "spark.yarn.queue"), OptionAssigner(args.numExecutors, YARN, ALL_DEPLOY_MODES, confKey = "spark.executor.instances"), diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 947d672c7fd56..a4e396849f3ea 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -694,15 +694,14 @@ private[spark] class Client( } } - val configDirProperty = sparkConf.getOption("spark.hadoop.config.dir") - // SPARK-23630: during testing, Spark scripts filter out hadoop conf dirs so that user's // environments do not interfere with tests. This allows a special env variable during // tests so that custom conf dirs can be used by unit tests. val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") ++ (if (Utils.isTesting) Seq("SPARK_TEST_HADOOP_CONF_DIR") else Nil) + val configDirProp = sparkConf.getOption("spark.hadoop.config.dir") - val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProperty).flatMap(_.toList) + val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProp).flatMap(_.toList) confDirPaths.foreach { path => logDebug("Reading config files from " + path) val dir = new File(path) From 5622511769a6422d6268163fa7e77981e904b732 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 30 Aug 2018 19:31:08 +0100 Subject: [PATCH 03/10] test --- .../scala/org/apache/spark/deploy/SparkSubmitArguments.scala | 4 ++-- .../org/apache/spark/launcher/AbstractCommandBuilder.java | 2 +- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../org/apache/spark/deploy/yarn/YarnClusterSuite.scala | 5 +++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 842460821ed22..77de0314f8f6b 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -286,11 +286,11 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S if (master.startsWith("yarn")) { val hasHadoopEnv = env.contains("HADOOP_CONF_DIR") || env.contains("YARN_CONF_DIR") - val hasHadoopProp = sparkProperties.contains("spark.yarn.config.dir") + val hasHadoopProp = sparkProperties.contains("spark.yarn.conf.dir") if (!hasHadoopEnv && !hasHadoopProp && !Utils.isTesting) { error(s"When running with master '$master' " + "either HADOOP_CONF_DIR or YARN_CONF_DIR must be set in the environment, +" + - "or spark.yarn.config.dir in the spark properties.") + "or spark.yarn.conf.dir in the spark properties.") } } diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index 9bde2192f8708..bc089f74267d8 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -199,7 +199,7 @@ List buildClassPath(String appClassPath) throws IOException { } addToClassPath(cp, getenv("HADOOP_CONF_DIR")); - addToClassPath(cp, getEffectiveConfig().get("spark.yarn.config.dir")); + addToClassPath(cp, getEffectiveConfig().get("spark.yarn.conf.dir")); addToClassPath(cp, getenv("YARN_CONF_DIR")); addToClassPath(cp, getenv("SPARK_DIST_CLASSPATH")); return new ArrayList<>(cp); diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index a4e396849f3ea..69b8e1c78852a 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -699,7 +699,7 @@ private[spark] class Client( // tests so that custom conf dirs can be used by unit tests. val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") ++ (if (Utils.isTesting) Seq("SPARK_TEST_HADOOP_CONF_DIR") else Nil) - val configDirProp = sparkConf.getOption("spark.hadoop.config.dir") + val configDirProp = sparkConf.getOption("spark.yarn.conf.dir") val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProp).flatMap(_.toList) confDirPaths.foreach { path => diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index 58d11e96942e1..eed3172aff3b2 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -131,8 +131,9 @@ class YarnClusterSuite extends BaseYarnClusterSuite { val finalState = runSpark(false, mainClassName(YarnClusterDriverUseSparkHadoopUtilConf.getClass), appArgs = Seq("key=value", "spark.test.key=testvalue", result.getAbsolutePath()), - extraConf = Map("spark.hadoop.key" -> "value"), - extraEnv = Map("SPARK_TEST_HADOOP_CONF_DIR" -> customConf.getAbsolutePath())) + extraConf = Map( + "spark.hadoop.key" -> "value", + "spark.yarn.config.dir" -> customConf.getAbsolutePath)) checkResult(finalState, result) } From d60736acc9b346c75669f924f34a29add855c53b Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 30 Aug 2018 19:39:26 +0100 Subject: [PATCH 04/10] remove SPARK_TEST_HADOOP_CONF_DIR --- .../main/scala/org/apache/spark/deploy/yarn/Client.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 69b8e1c78852a..8e5e7f563b836 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -694,11 +694,7 @@ private[spark] class Client( } } - // SPARK-23630: during testing, Spark scripts filter out hadoop conf dirs so that user's - // environments do not interfere with tests. This allows a special env variable during - // tests so that custom conf dirs can be used by unit tests. - val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") ++ - (if (Utils.isTesting) Seq("SPARK_TEST_HADOOP_CONF_DIR") else Nil) + val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") val configDirProp = sparkConf.getOption("spark.yarn.conf.dir") val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProp).flatMap(_.toList) From d690b26956323144f8eeb76e79467920ca7dde23 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 30 Aug 2018 19:40:17 +0100 Subject: [PATCH 05/10] move conf.dir to end of classpath --- .../java/org/apache/spark/launcher/AbstractCommandBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index bc089f74267d8..ead48d3df1c2c 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -199,8 +199,8 @@ List buildClassPath(String appClassPath) throws IOException { } addToClassPath(cp, getenv("HADOOP_CONF_DIR")); - addToClassPath(cp, getEffectiveConfig().get("spark.yarn.conf.dir")); addToClassPath(cp, getenv("YARN_CONF_DIR")); + addToClassPath(cp, getEffectiveConfig().get("spark.yarn.conf.dir")); addToClassPath(cp, getenv("SPARK_DIST_CLASSPATH")); return new ArrayList<>(cp); } From bed3f447886caa5419575d8cbc5f29e4ab8ab9cf Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 30 Aug 2018 19:45:06 +0100 Subject: [PATCH 06/10] use right prop in test --- .../scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index eed3172aff3b2..816e8a2138a28 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -133,7 +133,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite { appArgs = Seq("key=value", "spark.test.key=testvalue", result.getAbsolutePath()), extraConf = Map( "spark.hadoop.key" -> "value", - "spark.yarn.config.dir" -> customConf.getAbsolutePath)) + "spark.yarn.conf.dir" -> customConf.getAbsolutePath)) checkResult(finalState, result) } From f8e828c1c2495cfb55bb0142051c41389a3f876a Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Mon, 3 Sep 2018 17:13:18 +0100 Subject: [PATCH 07/10] rename to hadoop.conf.dir --- .../scala/org/apache/spark/deploy/SparkSubmitArguments.scala | 4 ++-- .../org/apache/spark/launcher/AbstractCommandBuilder.java | 2 +- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 77de0314f8f6b..3bdb088ac2253 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -286,11 +286,11 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S if (master.startsWith("yarn")) { val hasHadoopEnv = env.contains("HADOOP_CONF_DIR") || env.contains("YARN_CONF_DIR") - val hasHadoopProp = sparkProperties.contains("spark.yarn.conf.dir") + val hasHadoopProp = sparkProperties.contains("spark.hadoop.conf.dir") if (!hasHadoopEnv && !hasHadoopProp && !Utils.isTesting) { error(s"When running with master '$master' " + "either HADOOP_CONF_DIR or YARN_CONF_DIR must be set in the environment, +" + - "or spark.yarn.conf.dir in the spark properties.") + "or spark.hadoop.conf.dir in the spark properties.") } } diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index ead48d3df1c2c..d1af09c5bfa2d 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -200,7 +200,7 @@ List buildClassPath(String appClassPath) throws IOException { addToClassPath(cp, getenv("HADOOP_CONF_DIR")); addToClassPath(cp, getenv("YARN_CONF_DIR")); - addToClassPath(cp, getEffectiveConfig().get("spark.yarn.conf.dir")); + addToClassPath(cp, getEffectiveConfig().get("spark.hadoop.conf.dir")); addToClassPath(cp, getenv("SPARK_DIST_CLASSPATH")); return new ArrayList<>(cp); } diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 8e5e7f563b836..2ac2a62f6bc74 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -695,7 +695,7 @@ private[spark] class Client( } val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") - val configDirProp = sparkConf.getOption("spark.yarn.conf.dir") + val configDirProp = sparkConf.getOption("spark.hadoop.conf.dir") val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProp).flatMap(_.toList) confDirPaths.foreach { path => diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index 816e8a2138a28..8dd1e97ef4aed 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -133,7 +133,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite { appArgs = Seq("key=value", "spark.test.key=testvalue", result.getAbsolutePath()), extraConf = Map( "spark.hadoop.key" -> "value", - "spark.yarn.conf.dir" -> customConf.getAbsolutePath)) + "spark.hadoop.conf.dir" -> customConf.getAbsolutePath)) checkResult(finalState, result) } From 8bea68b10fa5ebe10ae17fe3ae012ec803106ba8 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Wed, 12 Sep 2018 18:29:11 +0100 Subject: [PATCH 08/10] Revert "rename to hadoop.conf.dir" This reverts commit f8e828c1c2495cfb55bb0142051c41389a3f876a. --- .../scala/org/apache/spark/deploy/SparkSubmitArguments.scala | 4 ++-- .../org/apache/spark/launcher/AbstractCommandBuilder.java | 2 +- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala index 3bdb088ac2253..77de0314f8f6b 100644 --- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala +++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala @@ -286,11 +286,11 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S if (master.startsWith("yarn")) { val hasHadoopEnv = env.contains("HADOOP_CONF_DIR") || env.contains("YARN_CONF_DIR") - val hasHadoopProp = sparkProperties.contains("spark.hadoop.conf.dir") + val hasHadoopProp = sparkProperties.contains("spark.yarn.conf.dir") if (!hasHadoopEnv && !hasHadoopProp && !Utils.isTesting) { error(s"When running with master '$master' " + "either HADOOP_CONF_DIR or YARN_CONF_DIR must be set in the environment, +" + - "or spark.hadoop.conf.dir in the spark properties.") + "or spark.yarn.conf.dir in the spark properties.") } } diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java index d1af09c5bfa2d..ead48d3df1c2c 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java @@ -200,7 +200,7 @@ List buildClassPath(String appClassPath) throws IOException { addToClassPath(cp, getenv("HADOOP_CONF_DIR")); addToClassPath(cp, getenv("YARN_CONF_DIR")); - addToClassPath(cp, getEffectiveConfig().get("spark.hadoop.conf.dir")); + addToClassPath(cp, getEffectiveConfig().get("spark.yarn.conf.dir")); addToClassPath(cp, getenv("SPARK_DIST_CLASSPATH")); return new ArrayList<>(cp); } diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 2ac2a62f6bc74..8e5e7f563b836 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -695,7 +695,7 @@ private[spark] class Client( } val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") - val configDirProp = sparkConf.getOption("spark.hadoop.conf.dir") + val configDirProp = sparkConf.getOption("spark.yarn.conf.dir") val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProp).flatMap(_.toList) confDirPaths.foreach { path => diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index 8dd1e97ef4aed..816e8a2138a28 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -133,7 +133,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite { appArgs = Seq("key=value", "spark.test.key=testvalue", result.getAbsolutePath()), extraConf = Map( "spark.hadoop.key" -> "value", - "spark.hadoop.conf.dir" -> customConf.getAbsolutePath)) + "spark.yarn.conf.dir" -> customConf.getAbsolutePath)) checkResult(finalState, result) } From 3680c12a8f6e17f195e7786d62994f0f797617b6 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Wed, 12 Sep 2018 18:31:37 +0100 Subject: [PATCH 09/10] rename --- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 8e5e7f563b836..66e34de861b87 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -695,9 +695,9 @@ private[spark] class Client( } val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") - val configDirProp = sparkConf.getOption("spark.yarn.conf.dir") + val confDirProp = sparkConf.getOption("spark.yarn.conf.dir") - val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ configDirProp).flatMap(_.toList) + val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ confDirProp).flatMap(_.toList) confDirPaths.foreach { path => logDebug("Reading config files from " + path) val dir = new File(path) From df45ff90f83979b167c5b28f5fc69e900e443c44 Mon Sep 17 00:00:00 2001 From: Adam Balogh Date: Thu, 13 Sep 2018 11:04:28 +0100 Subject: [PATCH 10/10] flatten + fix test --- .../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +- .../scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala index 66e34de861b87..33028d561b131 100644 --- a/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala +++ b/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala @@ -697,7 +697,7 @@ private[spark] class Client( val confDirsEnvKeys = Seq("HADOOP_CONF_DIR", "YARN_CONF_DIR") val confDirProp = sparkConf.getOption("spark.yarn.conf.dir") - val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ confDirProp).flatMap(_.toList) + val confDirPaths = (confDirsEnvKeys.map(sys.env.get) :+ confDirProp).flatten confDirPaths.foreach { path => logDebug("Reading config files from " + path) val dir = new File(path) diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala index 816e8a2138a28..e49d7c6001892 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/YarnClusterSuite.scala @@ -133,7 +133,7 @@ class YarnClusterSuite extends BaseYarnClusterSuite { appArgs = Seq("key=value", "spark.test.key=testvalue", result.getAbsolutePath()), extraConf = Map( "spark.hadoop.key" -> "value", - "spark.yarn.conf.dir" -> customConf.getAbsolutePath)) + "spark.yarn.conf.dir" -> customConf.getAbsolutePath())) checkResult(finalState, result) }