fix shared folders (apache#140)

jlopezmalla · Jan 30, 2018 · 6de89ec · 6de89ec
1 parent 08db386
commit 6de89ec
Show file tree

Hide file tree

Showing 3 changed files with 44 additions and 2 deletions.
diff --git a/core/src/main/scala/org/apache/spark/security/ConfigSecurity.scala b/core/src/main/scala/org/apache/spark/security/ConfigSecurity.scala
@@ -25,8 +25,16 @@ import org.apache.spark.util.Utils
 
 object ConfigSecurity extends Logging {
 
-  val secretsFolder = Utils.createTempDir(
-    s"${sys.env.getOrElse("SPARK_SECRETS_FOLDER", "/tmp")}", "spark").getAbsolutePath
+
+  val secretsFolder: String = sys.env.get("SPARK_DRIVER_SECRET_FOLDER") match {
+    case Some(folder) =>
+      logDebug(s"Creating secret folder using driver information in path $folder")
+      Utils.createDirectoryByName(folder).getAbsolutePath
+    case None =>
+      logDebug(s"Creating secret folder for executor")
+      Utils.createTempDir(
+       s"${sys.env.getOrElse("SPARK_SECRETS_FOLDER", "/tmp")}", "spark").getAbsolutePath
+  }
 
   lazy val vaultToken: Option[String] =
 

diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -313,6 +313,34 @@ private[spark] object Utils extends Logging {
     dir
   }
 
+  /**
+    * Create a temporary directory inside the given parent directory. The directory will be
+    * automatically deleted when the VM shuts down.
+    * It wont have an UUID identifier
+    */
+  def createDirectoryByName(root: String): File = {
+    var attempts = 0
+    val maxAttempts = MAX_DIR_CREATION_ATTEMPTS
+    var dir: File = null
+    while (dir == null) {
+      attempts += 1
+      if (attempts > maxAttempts) {
+        throw new IOException("Failed to create a temp directory (under " + root + ") after " +
+          maxAttempts + " attempts!")
+      }
+      try {
+        dir = new File(root)
+        if (dir.exists() || !dir.mkdirs()) {
+          dir = null
+        }
+      } catch { case e: SecurityException => dir = null; }
+    }
+
+    dir = dir.getCanonicalFile
+    ShutdownHookManager.registerShutdownDeleteDir(dir)
+    dir
+  }
+
   /**
    * Copy all data from an InputStream to an OutputStream. NIO way of file stream to file stream
    * copying is disabled by default unless explicitly set transferToEnabled as true,

diff --git a/...n/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala b/...n/scala/org/apache/spark/scheduler/cluster/mesos/MesosCoarseGrainedSchedulerBackend.scala
@@ -244,8 +244,14 @@ private[spark] class MesosCoarseGrainedSchedulerBackend(
         .setName("VAULT_PORT")
         .setValue(sys.env.get("VAULT_PORT").get)
         .build())
+      environment.addVariables(Environment.Variable.newBuilder()
+        .setName("SPARK_DRIVER_SECRET_FOLDER")
+        .setValue(ConfigSecurity.secretsFolder)
+        .build())
     }
 
+
+
     val command = CommandInfo.newBuilder()
       .setEnvironment(environment)