added Driver Memory Usage when you hove over Memory usage with alerts…

… to high and low utilization
dataflint · Sep 19, 2024 · af13c7a · af13c7a
1 parent 21ecb6b
commit af13c7a
Show file tree

Hide file tree

Showing 6 changed files with 66 additions and 72 deletions.
diff --git a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUILoader.scala b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/DataflintSparkUILoader.scala
@@ -43,8 +43,8 @@ class DataflintSparkUIInstaller extends Logging {
     }
 
     val runtime = Runtime.getRuntime
-    val driverXmxBytes = runtime.maxMemory()
-    val environmentInfo = DataflintEnvironmentInfo(driverXmxBytes)
+    val memoryUsagePercentage = (runtime.totalMemory() - runtime.freeMemory()).toDouble / runtime.totalMemory() * 100
+    val environmentInfo = DataflintEnvironmentInfo(memoryUsagePercentage)
     val isDatabricks = context.conf.getOption("spark.databricks.clusterUsageTags.cloudProvider").isDefined
     val icebergInstalled = context.conf.get("spark.sql.extensions", "").contains("org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions")
     val icebergEnabled = context.conf.getBoolean("spark.dataflint.iceberg.enabled", defaultValue = true)
@@ -81,8 +81,8 @@ class DataflintSparkUIInstaller extends Logging {
         else (listener: SparkListenerInterface, queue: String) => context.listenerBus.addToQueue(listener, queue)
       addToQueueMethod(dataflintListener, "dataflint")
       context.listenerBus.post(DataflintEnvironmentInfoEvent(environmentInfo))
-      logInfo(s"Posted DataflintEnvironmentInfoEvent with driver memory: $driverXmxBytes bytes")
-      if ((icebergInstalled && icebergEnabled) || isDatabricks) {
+      logInfo(s"Posted DataflintEnvironmentInfoEvent with driver memory usage: ${memoryUsagePercentage.round}%")
+      if (isDatabricks) {
         if (isDatabricks) {
           addToQueueMethod(DataflintDatabricksLiveListener(context.listenerBus), "dataflint")
           logInfo("Added DataflintDatabricksLiveListener to the Spark context")

diff --git a/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/listener/model.scala b/spark-plugin/plugin/src/main/scala/org/apache/spark/dataflint/listener/model.scala
@@ -55,7 +55,7 @@ class IcebergCommitWrapper(val info: IcebergCommitInfo) {
   def id: String = info.executionId.toString
 }
 
-case class DataflintEnvironmentInfo(driverXmxBytes: Long)
+case class DataflintEnvironmentInfo(driverMemoryUsagePercentage: Double)
 
 case class DataflintEnvironmentInfoEvent(environmentInfo: DataflintEnvironmentInfo) extends SparkListenerEvent
 

diff --git a/spark-ui/src/components/StatusBar.tsx b/spark-ui/src/components/StatusBar.tsx
@@ -48,7 +48,8 @@ const StatusBar: FC = (): JSX.Element => {
   );
   const durationText = humanizeTimeDiff(duration(status?.duration), true);
   const numOfQueries = sql?.sqls.length ?? 0;
-  const driverMemoryUsage = environmentInfo?.driverXmxBytes ?? 0;
+  const driverMemoryUsage = environmentInfo?.driverMemoryUsagePercentage ?? 0;
+
   const driverMemoryUsageString = humanFileSize(driverMemoryUsage);
   return (
     <div>
@@ -83,22 +84,26 @@ const StatusBar: FC = (): JSX.Element => {
             icon={MemoryIcon}
             tooltipContent={
               <React.Fragment>
-                <Typography variant="h6" color="inherit" textAlign={"center"}>
-                  {status?.executors?.maxExecutorMemoryBytesString} /{" "}
-                  {executorMemoryBytesString}
-                </Typography>
-                <Typography variant="subtitle2">
-                  Peak JVM memory usage / total executor JVM memory
-                </Typography>
-                <Typography variant="subtitle2">
-                  "Memory Usage" is the peak memory usage of the most
-                  memory-utilized executor.
-                </Typography>
-                <Typography variant="subtitle2">
-                  "Executor Memory" refers to the executor JVM memory (both
-                  on-heap and off-heap).
-                </Typography>
-              </React.Fragment>
+              <Typography variant="h6" color="inherit" textAlign={"center"}>
+                {status?.executors?.maxExecutorMemoryBytesString} /{" "}
+                {executorMemoryBytesString}
+              </Typography>
+              <Typography variant="subtitle2">
+                Peak Executor Memory: {status?.executors?.maxExecutorMemoryPercentage.toFixed(2)}%
+              </Typography>
+              <Typography variant="subtitle2">
+                Driver Memory Usage: {driverMemoryUsage.toFixed(2)}%
+              </Typography>
+              <Typography variant="subtitle2">
+                "Memory Usage" shows the peak memory usage of the most memory-utilized executor.
+              </Typography>
+              <Typography variant="subtitle2">
+                "Executor Memory" refers to the executor JVM memory (both on-heap and off-heap).
+              </Typography>
+              <Typography variant="subtitle2">
+                "Driver Memory Usage" shows the current memory usage of the Spark driver as a percentage of its maximum allocated memory.
+              </Typography>
+            </React.Fragment>
             }
           ></InfoBox>
           <InfoBox
@@ -113,22 +118,6 @@ const StatusBar: FC = (): JSX.Element => {
             color="#00838f"
             icon={DatasetIcon}
           ></InfoBox>
-          <InfoBox
-            title="Driver Memory"
-            text={driverMemoryUsageString}
-            color="#8e24aa"
-            icon={MemoryIcon}
-            tooltipContent={
-              <React.Fragment>
-                <Typography variant="h6" color="inherit" textAlign={"center"}>
-                  {driverMemoryUsageString}
-                </Typography>
-                <Typography variant="subtitle2">
-                This represents the total JVM heap memory allocated to the Spark driver process. The actual available memory for Spark tasks is typically slightly less due to system overhead.
-                </Typography>
-              </React.Fragment>
-            }
-          ></InfoBox>
         </Grid>
       ) : (
         <Grid

diff --git a/spark-ui/src/components/SummaryBar.tsx b/spark-ui/src/components/SummaryBar.tsx
@@ -12,14 +12,16 @@ import { duration } from "moment";
 import React, { FC } from "react";
 import "reactflow/dist/style.css";
 import { useAppSelector } from "../Hooks";
+import { humanFileSize, humanizeTimeDiff } from "../utils/FormatUtils";
 import InfoBox from "./InfoBox/InfoBox";
 import Progress from "./Progress";
-import { humanFileSize, humanizeTimeDiff } from "../utils/FormatUtils";
 
 const SummaryBar: FC = (): JSX.Element => {
   const status = useAppSelector((state) => state.spark.status);
   const alerts = useAppSelector((state) => state.spark.alerts);
-  const environmentInfo = useAppSelector((state) => state.spark.environmentInfo);
+  const environmentInfo = useAppSelector(
+    (state) => state.spark.environmentInfo,
+  );
   const memoryAlert = alerts?.alerts.find(
     (alert) =>
       alert.source.type === "status" && alert.source.metric === "memory",
@@ -41,7 +43,7 @@ const SummaryBar: FC = (): JSX.Element => {
   }
 
   const durationText = humanizeTimeDiff(duration(status.duration));
-  const driverMemoryUsage = environmentInfo?.driverXmxBytes ?? 0;
+  const driverMemoryUsage = environmentInfo?.driverMemoryUsagePercentage ?? 0;
   const driverMemoryUsageString = humanFileSize(driverMemoryUsage);
   const totalDCUFormated =
     status.executors.totalDCU > 1
@@ -131,35 +133,27 @@ const SummaryBar: FC = (): JSX.Element => {
                 {executorMemoryBytesString}
               </Typography>
               <Typography variant="subtitle2">
-                Peak JVM memory usage / total executor JVM memory
+                Peak Executor Memory:{" "}
+                {status.executors.maxExecutorMemoryPercentage.toFixed(2)}%
               </Typography>
               <Typography variant="subtitle2">
-                "Memory Usage" is the peak memory usage of the most
+                Driver Memory Usage: {driverMemoryUsage.toFixed(2)}%
+              </Typography>
+              <Typography variant="subtitle2">
+                "Memory Usage" shows the peak memory usage of the most
                 memory-utilized executor.
               </Typography>
               <Typography variant="subtitle2">
                 "Executor Memory" refers to the executor JVM memory (both
                 on-heap and off-heap).
               </Typography>
+              <Typography variant="subtitle2">
+                "Driver Memory Usage" shows the current memory usage of the
+                Spark driver as a percentage of its maximum allocated memory.
+              </Typography>
             </React.Fragment>
           }
         ></InfoBox>
-                  <InfoBox
-            title="Driver Memory"
-            text={driverMemoryUsageString}
-            color="#8e24aa"
-            icon={MemoryIcon}
-            tooltipContent={
-              <React.Fragment>
-                <Typography variant="h6" color="inherit" textAlign={"center"}>
-                  {driverMemoryUsageString}
-                </Typography>
-                <Typography variant="subtitle2">
-                This represents the total JVM heap memory allocated to the Spark driver process. The actual available memory for Spark tasks is typically slightly less due to system overhead.
-                </Typography>
-              </React.Fragment>
-            }
-          ></InfoBox>
       </Grid>
       <Grid
         container

diff --git a/spark-ui/src/interfaces/ApplicationInfo.ts b/spark-ui/src/interfaces/ApplicationInfo.ts
@@ -1,7 +1,7 @@
 import { SparkApplication } from "./SparkApplications";
 
 export interface EnvironmentInfo {
-  driverXmxBytes: number;
+  driverMemoryUsagePercentage: number;
 }
 
 export interface ApplicationInfo {

diff --git a/spark-ui/src/reducers/Alerts/MemoryAlertsReducer.ts b/spark-ui/src/reducers/Alerts/MemoryAlertsReducer.ts
@@ -5,9 +5,10 @@ import { humanFileSize } from '../../utils/FormatUtils';
 
 const MAX_EXECUTOR_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD = 95;
 const MAX_EXECUTOR_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD = 70;
+const MAX_DRIVER_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD = 90;
+const MAX_DRIVER_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD = 10;
 const MEMORY_INCRESE_RATIO = 0.2;
 const MEMORY_DECREASE_SAFETRY_BUFFER = 0.2;
-const MIN_DRIVER_MEMORY = 1073741824; 
 
 export function reduceMemoryAlerts(
   statusStore: StatusStore,
@@ -83,24 +84,34 @@ export function reduceMemoryAlerts(
   reduceDriverMemoryAlerts(statusStore, config, environmentInfo, alerts);
 }
 
+
 function reduceDriverMemoryAlerts(
   statusStore: StatusStore,
   config: ConfigStore,
   environmentInfo: any,
   alerts: Alerts
 ) {
-  const driverMemory = environmentInfo?.driverXmxBytes ?? 0;
-  const totalInputBytes = statusStore.executors?.totalInputBytes ?? 0;
-  const numCores = parseInt(config.configs.find(c => c.key === "spark.executor.cores")?.value ?? "1");
-  if (driverMemory < MIN_DRIVER_MEMORY) {
+  const driverMemoryUsage = environmentInfo?.driverMemoryUsagePercentage ?? 0;
+
+  if (driverMemoryUsage > MAX_DRIVER_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD) {
+    alerts.push({
+      id: `driverMemoryTooHigh_${uuidv4()}`,
+      name: "driverMemoryTooHigh",
+      title: "Driver Memory Usage Too High",
+      message: `The Spark driver memory usage is ${driverMemoryUsage.toFixed(2)}%, which is over ${MAX_DRIVER_MEMORY_PERCENTAGE_TOO_HIGH_THRESHOLD}% and may lead to out-of-memory errors.`,
+      location: "In: Spark Driver",
+      suggestion: `Consider increasing the driver memory`,
+      type: "warning",
+      source: { type: "status", metric: "memory" }
+    });
+  } else if (driverMemoryUsage < MAX_DRIVER_MEMORY_PERCENTAGE_TOO_LOW_THRESHOLD) {
     alerts.push({
-      id: `driverMemoryTooSmall_${uuidv4()}`,
-      name: "driverMemoryTooSmall",
-      title: "Driver Memory Too Small",
-      message: "The Spark driver memory is set to less than 1GB, which may be insufficient for most workloads.",
-      location: `In: Spark Driver (Current llocated memory is: ${humanFileSize(driverMemory)})`,
-      suggestion: `Consider increasing the driver memory to at least 2GB by adding this configuration:
-.config("spark.driver.memory", "2g")`,
+      id: `driverMemoryTooLow_${uuidv4()}`,
+      name: "driverMemoryTooLow",
+      title: "Driver Memory Usage Too Low",
+      message: `The Spark driver memory usage is only ${driverMemoryUsage.toFixed(2)}%, which suggests over-allocation of driver memory.`,
+      location: "In: Spark Driver",
+      suggestion: `Consider decreasing the driver memory to save resources`,
       type: "warning",
       source: { type: "status", metric: "memory" }
     });