Merge pull request apache#5 from glorysdj/pr1

add sgx log level option
qiuxin2012 · Dec 2, 2021 · 5783d82 · 5783d82
2 parents 6acc3e4 + c0c862c
commit 5783d82
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 4 deletions.
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -80,6 +80,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
   var sgxEnabled = false
   var sgxMem: String = null
   var sgxJvmMem: String = null
+  var sgxLogLevel: String = null
 
   // Standalone cluster mode only
   var supervise: Boolean = false
@@ -220,6 +221,8 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
       .getOrElse(sparkProperties.get(config.SGX_MEM_SIZE.key).orNull)
     sgxJvmMem = Option(sgxJvmMem)
       .getOrElse(sparkProperties.get(config.SGX_JVM_MEM_SIZE.key).orNull)
+    sgxLogLevel = Option(sgxLogLevel)
+      .getOrElse(sparkProperties.get(config.SGX_JVM_LOG_LEVEL.key).orNull)
 
     // Global defaults. These should be keep to minimum to avoid confusing behavior.
     master = Option(master).getOrElse("local[*]")
@@ -335,6 +338,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
     |  sgxEnabled              $sgxEnabled
     |  sgxMem                  $sgxMem
     |  sgxJvmMem               $sgxJvmMem
+    |  sgxLogLevel             $sgxLogLevel
     |
     |Spark properties used, including those specified through
     | --conf and those from the properties file $propertiesFile:

diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -2044,4 +2044,11 @@ package object config {
       .version("3.1.2")
       .bytesConf(ByteUnit.GiB)
       .createWithDefaultString("16g")
+
+  private[spark] val SGX_JVM_LOG_LEVEL =
+    ConfigBuilder("spark.kubernetes.sgx.log.level")
+      .doc("the log level of sgx execuotrs, e.g. all, error, debug")
+      .version("3.1.2")
+      .stringConf
+      .createWithDefaultString("error")
 }
diff --git a/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala b/core/src/main/scala/org/apache/spark/resource/ResourceProfile.scala
@@ -356,9 +356,10 @@ object ResourceProfile extends Logging {
     val sgxEnabled = conf.get(SGX_ENABLED)
     val sgxMem = conf.get(SGX_MEM_SIZE)
     val sgxJvmMem = conf.get(SGX_JVM_MEM_SIZE)
+    val sgxLogLevel = conf.get(SGX_JVM_LOG_LEVEL)
     defaultProfileExecutorResources =
       Some(DefaultProfileExecutorResources(cores, memory, offheapMem, pysparkMem,
-        overheadMem, customResources, sgxEnabled, sgxMem, sgxJvmMem))
+        overheadMem, customResources, sgxEnabled, sgxMem, sgxJvmMem, sgxLogLevel))
     ereqs.requests
   }
 
@@ -414,7 +415,8 @@ object ResourceProfile extends Logging {
       customResources: Map[String, ExecutorResourceRequest],
       sgxEnabled: Boolean,
       sgxMemGiB: Long,
-      sgxJvmMemGiB: Long)
+      sgxJvmMemGiB: Long,
+      sgxLogLevel: String)
 
   private[spark] case class DefaultProfileExecutorResources(
       cores: Int,
@@ -425,7 +427,8 @@ object ResourceProfile extends Logging {
       customResources: Map[String, ExecutorResourceRequest],
       sgxEnabled: Boolean,
       sgxMemGiB: Long,
-      sgxJvmMemGiB: Long)
+      sgxJvmMemGiB: Long,
+      sgxLogLevel: String)
 
   private[spark] def calculateOverHeadMemory(
       overHeadMemFromConf: Option[Long],
@@ -495,9 +498,10 @@ object ResourceProfile extends Logging {
     val sgxEnabled = defaultResources.sgxEnabled
     val sgxMemGiB = defaultResources.sgxMemGiB
     val sgxJvmMemGiB = defaultResources.sgxJvmMemGiB
+    val sgxLogLevel = defaultResources.sgxLogLevel
     ExecutorResourcesOrDefaults(cores, executorMemoryMiB, memoryOffHeapMiB,
       pysparkMemToUseMiB, memoryOverheadMiB, totalMemMiB, finalCustomResources,
-      sgxEnabled, sgxMemGiB, sgxJvmMemGiB)
+      sgxEnabled, sgxMemGiB, sgxJvmMemGiB, sgxLogLevel)
   }
 
   private[spark] val PYSPARK_MEMORY_LOCAL_PROPERTY = "resource.pyspark.memory"