From c6530280859ef8ec8419c89fca94dd9e9d866506 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Thu, 6 Nov 2014 21:04:11 -0800 Subject: [PATCH 1/5] Package network-yarn and its dependencies This allows make-distribution to create a small uber jar for the network-yarn module, such that all uses of the Yarn shuffle service can just drop this jar onto the NM classpath and start the shuffle service after configuring the NM to include it. --- make-distribution.sh | 1 + network/common/pom.xml | 4 ++-- network/shuffle/pom.xml | 4 ++-- network/yarn/pom.xml | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index 0bc839e1dbe4d..ad5e4f60e234e 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -181,6 +181,7 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI # Copy jars cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/" cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/" +cp "$FWDIR"/network/yarn/target/scala*/spark-network-yarn*.jar "$DISTDIR/lib/" # Copy example sources (needed for python and SQL) mkdir -p "$DISTDIR/examples/src/main" diff --git a/network/common/pom.xml b/network/common/pom.xml index 6144548a8f998..89ba964fa5623 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -41,12 +41,12 @@ io.netty netty-all + + org.slf4j slf4j-api - - com.google.guava guava diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index fe5681d463499..9f9155a62e483 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -42,12 +42,12 @@ spark-network-common_2.10 ${project.version} + + org.slf4j slf4j-api - - com.google.guava guava diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index e60d8c1f7876c..0bda52614c431 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -54,5 +54,38 @@ target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes + + + org.apache.maven.plugins + maven-shade-plugin + + false + ${project.build.directory}/scala-${scala.binary.version}/spark-network-yarn-${project.version}-hadoop${hadoop.version}.jar + + + *:* + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + package + + shade + + + + + From abcefd168baa2b5d73ca6c637ff0168338efc191 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Thu, 6 Nov 2014 21:07:18 -0800 Subject: [PATCH 2/5] Do the same for SBT --- project/SparkBuild.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 657e4b4432775..0ea3da99c0578 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -38,12 +38,12 @@ object BuildCommons { "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter", "streaming-zeromq").map(ProjectRef(buildLocation, _)) - val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, networkYarn, java8Tests, - sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "network-yarn", + val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, + sparkGangliaLgpl, sparkKinesisAsl) = Seq("yarn", "yarn-stable", "yarn-alpha", "java8-tests", "ganglia-lgpl", "kinesis-asl").map(ProjectRef(buildLocation, _)) - val assemblyProjects@Seq(assembly, examples) = Seq("assembly", "examples") - .map(ProjectRef(buildLocation, _)) + val assemblyProjects@Seq(assembly, examples, networkYarn) = + Seq("assembly", "examples", "network-yarn").map(ProjectRef(buildLocation, _)) val tools = ProjectRef(buildLocation, "tools") // Root project. From 65db8227ef5632ff53574fc8efd7c579b6f26133 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Fri, 7 Nov 2014 09:26:29 -0800 Subject: [PATCH 3/5] Actually mark slf4j as provided --- network/common/pom.xml | 1 + network/shuffle/pom.xml | 1 + 2 files changed, 2 insertions(+) diff --git a/network/common/pom.xml b/network/common/pom.xml index 89ba964fa5623..25f505c1bf905 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -46,6 +46,7 @@ org.slf4j slf4j-api + provided com.google.guava diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml index 9f9155a62e483..b158eb82a2fef 100644 --- a/network/shuffle/pom.xml +++ b/network/shuffle/pom.xml @@ -47,6 +47,7 @@ org.slf4j slf4j-api + provided com.google.guava From fb7f3982ba28c1cc4215f042cf2f37a75f188ba7 Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Mon, 10 Nov 2014 20:12:58 -0800 Subject: [PATCH 4/5] Rename jar to spark-{VERSION}-yarn-shuffle.jar --- make-distribution.sh | 2 +- network/yarn/pom.xml | 2 +- project/SparkBuild.scala | 10 ++++++++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/make-distribution.sh b/make-distribution.sh index ad5e4f60e234e..8e25c2ef479a3 100755 --- a/make-distribution.sh +++ b/make-distribution.sh @@ -181,7 +181,7 @@ echo "Spark $VERSION$GITREVSTRING built for Hadoop $SPARK_HADOOP_VERSION" > "$DI # Copy jars cp "$FWDIR"/assembly/target/scala*/*assembly*hadoop*.jar "$DISTDIR/lib/" cp "$FWDIR"/examples/target/scala*/spark-examples*.jar "$DISTDIR/lib/" -cp "$FWDIR"/network/yarn/target/scala*/spark-network-yarn*.jar "$DISTDIR/lib/" +cp "$FWDIR"/network/yarn/target/scala*/spark-*-yarn-shuffle.jar "$DISTDIR/lib/" # Copy example sources (needed for python and SQL) mkdir -p "$DISTDIR/examples/src/main" diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml index 0bda52614c431..3a25fcd4183fd 100644 --- a/network/yarn/pom.xml +++ b/network/yarn/pom.xml @@ -60,7 +60,7 @@ maven-shade-plugin false - ${project.build.directory}/scala-${scala.binary.version}/spark-network-yarn-${project.version}-hadoop${hadoop.version}.jar + ${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar *:* diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 0ea3da99c0578..16a532a98ea50 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -270,8 +270,14 @@ object Assembly { lazy val settings = assemblySettings ++ Seq( test in assembly := {}, - jarName in assembly <<= (version, moduleName) map { (v, mName) => mName + "-"+v + "-hadoop" + - Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" }, + jarName in assembly <<= (version, moduleName) map { (v, mName) => + if (mName.contains("network-yarn")) { + // This must match the same name used in maven (see network/yarn/pom.xml) + "spark-" + v + "-yarn-shuffle.jar" + } else { + mName + "-" + v + "-hadoop" + Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" + } + }, mergeStrategy in assembly := { case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard From bda58d09d3096dd1f60c91a816dba4ae3ce3159e Mon Sep 17 00:00:00 2001 From: Andrew Or Date: Mon, 10 Nov 2014 20:16:12 -0800 Subject: [PATCH 5/5] Fix line too long --- project/SparkBuild.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 16a532a98ea50..3db568c2b32da 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -275,7 +275,8 @@ object Assembly { // This must match the same name used in maven (see network/yarn/pom.xml) "spark-" + v + "-yarn-shuffle.jar" } else { - mName + "-" + v + "-hadoop" + Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" + mName + "-" + v + "-hadoop" + + Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" } }, mergeStrategy in assembly := {