Adding support for downloading logs directly from Spark UI (apache#23)

curtishoward · Sep 20, 2017 · 38fa491 · 38fa491
1 parent 2b5b8cd
commit 38fa491
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/cook/src/main/scala/org/apache/spark/scheduler/cluster/cook/CoarseCookSchedulerBackend.scala b/cook/src/main/scala/org/apache/spark/scheduler/cluster/cook/CoarseCookSchedulerBackend.scala
@@ -306,13 +306,20 @@ class CoarseCookSchedulerBackend(
       s"[ -z $$KEEP_SPARK_LOCAL_TARS ] || rm -f $$(basename $uri)"
     }
 
+    val logUrlCommand = Seq(
+      "export BASE_SPARK_LOG_URL=\"http://`hostname`:5051/files/download?path=$MESOS_SANDBOX\"",
+      "export SPARK_LOG_URL_STDERR=\"$BASE_SPARK_LOG_URL/stderr\"",
+      "export SPARK_LOG_URL_STDOUT=\"$BASE_SPARK_LOG_URL/stdout\""
+    )
+
     val commandSeq =
       debugCommand ++
         envSettingCommand ++
         uriFetchCommand ++
         shippedTarballsCommand ++
         remoteConfFetchCommand ++
         keystorePullCommand.map(Seq(_)).getOrElse(Seq[String]()) ++
+        logUrlCommand ++
         Seq("set", commandString) ++
         cleanup