Add min, median and max columns to AccumProfileResults

core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/ClassWarehouse.scala

@@ -187,19 +187,20 @@ class SQLExecutionInfoClass(
     var sqlCpuTimePercent: Double = -1)
 
 case class SQLAccumProfileResults(appIndex: Int, sqlID: Long, nodeID: Long,
-    nodeName: String, accumulatorId: Long, name: String, max_value: Long,
-    metricType: String, stageIds: String) extends ProfileResult {
+    nodeName: String, accumulatorId: Long, name: String, min: Long, median:Long,
+    max: Long, total: Long, metricType: String, stageIds: String) extends ProfileResult {
   override val outputHeaders = Seq("appIndex", "sqlID", "nodeID", "nodeName", "accumulatorId",
-    "name", "max_value", "metricType", "stageIds")
+    "name", "min", "median", "max", "total", "metricType", "stageIds")
   override def convertToSeq: Seq[String] = {
     Seq(appIndex.toString, sqlID.toString, nodeID.toString, nodeName, accumulatorId.toString,
-      name, max_value.toString, metricType, stageIds)
+      name, min.toString, median.toString, max.toString, total.toString, metricType, stageIds)
   }
   override def convertToCSVSeq: Seq[String] = {
     Seq(appIndex.toString, sqlID.toString, nodeID.toString,
       StringUtils.reformatCSVString(nodeName), accumulatorId.toString,
-      StringUtils.reformatCSVString(name), max_value.toString,
-      StringUtils.reformatCSVString(metricType), StringUtils.reformatCSVString(stageIds))
+      StringUtils.reformatCSVString(name), min.toString, median.toString, max.toString,
+      total.toString, StringUtils.reformatCSVString(metricType),
+      StringUtils.reformatCSVString(stageIds))
   }
 }
 

core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/CollectInformation.scala

@@ -22,6 +22,7 @@ import com.nvidia.spark.rapids.tool.ToolTextFileWriter
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.resource.ResourceProfile
+import org.apache.spark.sql.rapids.tool.SQLMetricsStats
 import org.apache.spark.sql.rapids.tool.profiling.ApplicationInfo
 
 case class StageMetrics(numTasks: Int, duration: String)
@@ -100,10 +101,10 @@ class CollectInformation(apps: Seq[ApplicationInfo]) extends Logging {
     def getIoMetrics(sqlAccums: Seq[SQLAccumProfileResults]): IoMetrics = {
       val finalRes = IoMetrics(0, 0, 0, 0)
       sqlAccums.map(accum => accum.name match {
-        case `buffer_time` => finalRes.buffer_time = accum.max_value
-        case `scan_time` => finalRes.scan_time = accum.max_value
-        case `data_size` => finalRes.data_size = accum.max_value
-        case `decode_time` => finalRes.decode_time = accum.max_value
+        case `buffer_time` => finalRes.buffer_time = accum.total
+        case `scan_time` => finalRes.scan_time = accum.total
+        case `data_size` => finalRes.data_size = accum.total
+        case `decode_time` => finalRes.decode_time = accum.total
       })
       finalRes
     }
@@ -261,6 +262,9 @@ class CollectInformation(apps: Seq[ApplicationInfo]) extends Logging {
 
 object CollectInformation extends Logging {
 
+ // Store (min, median, max, total) for a given metric
+  case class statisticsMetrics(min: Long, med:Long, max:Long, total: Long)
+
   def generateSQLAccums(apps: Seq[ApplicationInfo]): Seq[SQLAccumProfileResults] = {
     val allRows = apps.flatMap { app =>
       app.allSQLMetrics.map { metric =>
@@ -276,11 +280,26 @@ object CollectInformation extends Logging {
             val filtered = accums.filter { a =>
               stageIdsForSQL.contains(a.stageId)
             }
-            val accumValues = filtered.map(_.value.getOrElse(0L))
-            if (accumValues.isEmpty) {
-              None
+            // If metricType is size, average or timing, we want to read field `update` value
+            // to get the min, median, max, and total. Otherwise, we want to use field `value`.
+            if (SQLMetricsStats.hasStats(metric.metricType)) {
+              val accumValues = filtered.map(_.update.getOrElse(0L)).sortWith(_ < _)
+              if (accumValues.isEmpty) {
+                None
+              }
+              else if (accumValues.length <= 1) {
+                Some(statisticsMetrics(0L, 0L, 0L, accumValues.sum))
+              } else {
+                Some(statisticsMetrics(accumValues(0), accumValues(accumValues.size / 2),
+                  accumValues(accumValues.size - 1), accumValues.sum))
+              }
             } else {
-              Some(accumValues.max)
+              val accumValues = filtered.map(_.value.getOrElse(0L))
+              if (accumValues.isEmpty) {
+                None
+              } else {
+                Some(statisticsMetrics(0L, 0L, 0L, accumValues.max))
+              }
             }
           case None => None
         }
@@ -292,21 +311,37 @@ object CollectInformation extends Logging {
             val filtered = accums.filter { a =>
               a.sqlID == sqlId
             }
-            val accumValues = filtered.map(_.value)
+            val accumValues = filtered.map(_.value).sortWith(_ < _)
             if (accumValues.isEmpty) {
               None
+            } else if (accumValues.length <= 1) {
+              Some(statisticsMetrics(0L, 0L, 0L, accumValues.sum))
             } else {
-              Some(accumValues.max)
+              Some(statisticsMetrics(accumValues(0), accumValues(accumValues.size / 2),
+                accumValues(accumValues.size - 1), accumValues.sum))
             }
           case None =>
             None
         }
 
-        if ((taskMax.isDefined) || (driverMax.isDefined)) {
-          val max = Math.max(driverMax.getOrElse(0L), taskMax.getOrElse(0L))
+        if (taskMax.isDefined || driverMax.isDefined) {
+          val taskInfo = taskMax match {
+            case Some(task) => task
+            case None => statisticsMetrics(0L, 0L, 0L, 0L)
+          }
+          val driverInfo = driverMax match {
+            case Some(driver) => driver
+            case None => statisticsMetrics(0L, 0L, 0L, 0L)
+          }
+
+          val max = Math.max(taskInfo.max, driverInfo.max)
+          val min = Math.max(taskInfo.min, driverInfo.min)
+          val med = Math.max(taskInfo.med, driverInfo.med)
+          val total = Math.max(taskInfo.total, driverInfo.total)
+
           Some(SQLAccumProfileResults(app.index, metric.sqlID,
-            metric.nodeID, metric.nodeName, metric.accumulatorId,
-            metric.name, max, metric.metricType, metric.stageIds.mkString(",")))
+            metric.nodeID, metric.nodeName, metric.accumulatorId, metric.name,
+            min, med, max, total, metric.metricType, metric.stageIds.mkString(",")))
         } else {
           None
         }

core/src/main/scala/com/nvidia/spark/rapids/tool/profiling/GenerateDot.scala

@@ -86,7 +86,7 @@ object GenerateDot {
   def apply(app: ApplicationInfo, outputDirectory: String): Unit = {
     val accums = CollectInformation.generateSQLAccums(Seq(app))
     val accumSummary = accums.map { a =>
-      Seq(a.sqlID, a.accumulatorId, a.max_value)
+      Seq(a.sqlID, a.accumulatorId, a.total)
     }
     val accumIdToStageId = app.accumIdToStageId
     val formatter = java.text.NumberFormat.getIntegerInstance

core/src/main/scala/org/apache/spark/sql/rapids/tool/ToolUtils.scala

@@ -302,6 +302,21 @@ object BuildSide {
     BuildRight -> JoinType.supportedJoinTypeForBuildRight)
 }
 
+object SQLMetricsStats {
+  val SIZE_METRIC = "size"
+  val TIMING_METRIC = "timing"
+  val NS_TIMING_METRIC = "nsTiming"
+  val AVERAGE_METRIC = "average"
+  val SUM_METRIC = "sum"
+
+  def hasStats(metrics : String): Boolean = {
+    metrics match {
+      case SIZE_METRIC | TIMING_METRIC | NS_TIMING_METRIC | AVERAGE_METRIC => true
+      case _ => false
+    }
+  }
+}
+
 object MlOps {
   val sparkml = "spark.ml."
   val xgBoost = "spark.XGBoost"

core/src/test/resources/ProfilingExpectations/rapids_join_eventlog_sqlmetrics_expectation.csv

@@ -1,84 +1,84 @@
-appIndex,sqlID,nodeID,nodeName,accumulatorId,name,max_value,metricType,stageIds
-1,0,0,"GpuColumnarToRow",33,"total time",857404,"nsTiming","3"
-1,0,1,"GpuHashAggregate",34,"output rows",1,"sum","3"
-1,0,1,"GpuHashAggregate",35,"output columnar batches",1,"sum","3"
-1,0,1,"GpuHashAggregate",36,"total time",4212819,"nsTiming","3"
-1,0,1,"GpuHashAggregate",37,"aggregation time",3846803,"nsTiming","3"
-1,0,2,"GpuShuffleCoalesce",39,"output rows",200,"sum","3"
-1,0,2,"GpuShuffleCoalesce",40,"output columnar batches",1,"sum","3"
-1,0,2,"GpuShuffleCoalesce",41,"total time",3803240,"nsTiming","3"
-1,0,2,"GpuShuffleCoalesce",42,"collect batch time",3277904,"nsTiming","3"
-1,0,2,"GpuShuffleCoalesce",43,"concat batch time",392509,"nsTiming","3"
-1,0,3,"GpuColumnarExchange",44,"partition data size",16000,"sum","2,3"
-1,0,3,"GpuColumnarExchange",45,"partitions",1,"sum","2,3"
-1,0,3,"GpuColumnarExchange",46,"output rows",200,"sum","2,3"
-1,0,3,"GpuColumnarExchange",47,"output columnar batches",200,"sum","2,3"
-1,0,3,"GpuColumnarExchange",48,"data size",19600,"size","2,3"
-1,0,3,"GpuColumnarExchange",50,"local blocks read",200,"sum","2,3"
-1,0,3,"GpuColumnarExchange",53,"local bytes read",15400,"size","2,3"
-1,0,3,"GpuColumnarExchange",54,"fetch wait time",0,"timing","2,3"
-1,0,3,"GpuColumnarExchange",55,"records read",200,"sum","2,3"
-1,0,3,"GpuColumnarExchange",56,"shuffle bytes written",15400,"size","2,3"
-1,0,3,"GpuColumnarExchange",57,"shuffle records written",200,"sum","2,3"
-1,0,3,"GpuColumnarExchange",58,"shuffle write time",93193331,"nsTiming","2,3"
-1,0,4,"GpuHashAggregate",59,"output rows",200,"sum","2"
-1,0,4,"GpuHashAggregate",60,"output columnar batches",200,"sum","2"
-1,0,4,"GpuHashAggregate",61,"total time",80781515,"nsTiming","2"
-1,0,4,"GpuHashAggregate",62,"aggregation time",31923387,"nsTiming","2"
-1,0,5,"GpuProject",64,"total time",5377158,"nsTiming","2"
-1,0,6,"GpuShuffledHashJoin",65,"output rows",10000000,"sum","2"
-1,0,6,"GpuShuffledHashJoin",66,"output columnar batches",200,"sum","2"
-1,0,6,"GpuShuffledHashJoin",67,"total time",3904332009,"nsTiming","2"
-1,0,6,"GpuShuffledHashJoin",68,"build side size",80000000,"size","2"
-1,0,6,"GpuShuffledHashJoin",69,"build time",3448606506,"nsTiming","2"
-1,0,6,"GpuShuffledHashJoin",70,"stream time",260796041,"nsTiming","2"
-1,0,6,"GpuShuffledHashJoin",71,"join time",178084313,"nsTiming","2"
-1,0,6,"GpuShuffledHashJoin",72,"join output rows",10000000,"sum","2"
-1,0,7,"GpuShuffleCoalesce",74,"output rows",10000000,"sum","2"
-1,0,7,"GpuShuffleCoalesce",75,"output columnar batches",200,"sum","2"
-1,0,7,"GpuShuffleCoalesce",76,"total time",261389422,"nsTiming","2"
-1,0,7,"GpuShuffleCoalesce",77,"collect batch time",167775821,"nsTiming","2"
-1,0,7,"GpuShuffleCoalesce",78,"concat batch time",83550919,"nsTiming","2"
-1,0,8,"GpuColumnarExchange",79,"partition data size",42872100,"sum","1,2"
-1,0,8,"GpuColumnarExchange",80,"partitions",200,"sum","1,2"
-1,0,8,"GpuColumnarExchange",81,"output rows",10000000,"sum","1,2"
-1,0,8,"GpuColumnarExchange",82,"output columnar batches",1200,"sum","1,2"
-1,0,8,"GpuColumnarExchange",83,"data size",40076192,"size","1,2"
-1,0,8,"GpuColumnarExchange",85,"local blocks read",1200,"sum","1,2"
-1,0,8,"GpuColumnarExchange",88,"local bytes read",40132258,"size","1,2"
-1,0,8,"GpuColumnarExchange",89,"fetch wait time",0,"timing","1,2"
-1,0,8,"GpuColumnarExchange",90,"records read",1200,"sum","1,2"
-1,0,8,"GpuColumnarExchange",91,"shuffle bytes written",40132258,"size","1,2"
-1,0,8,"GpuColumnarExchange",92,"shuffle records written",1200,"sum","1,2"
-1,0,8,"GpuColumnarExchange",93,"shuffle write time",508750471,"nsTiming","1,2"
-1,0,9,"GpuProject",94,"total time",6667140,"nsTiming","1"
-1,0,10,"GpuRowToColumnar",95,"total time",61112304,"nsTiming","1"
-1,0,11,"WholeStageCodegen (1)",96,"duration",5463,"timing","1"
-1,0,13,"Scan",97,"number of output rows",10000000,"sum","1"
-1,0,14,"GpuCoalesceBatches",98,"output rows",10000000,"sum","2"
-1,0,14,"GpuCoalesceBatches",99,"output columnar batches",200,"sum","2"
-1,0,14,"GpuCoalesceBatches",100,"total time",3383354389,"nsTiming","2"
-1,0,14,"GpuCoalesceBatches",101,"collect batch time",3275108263,"nsTiming","2"
-1,0,14,"GpuCoalesceBatches",102,"concat batch time",20312708,"nsTiming","2"
-1,0,14,"GpuCoalesceBatches",103,"peak device memory",80000000,"size","2"
-1,0,15,"GpuShuffleCoalesce",107,"output rows",10000000,"sum","2"
-1,0,15,"GpuShuffleCoalesce",108,"output columnar batches",200,"sum","2"
-1,0,15,"GpuShuffleCoalesce",109,"total time",3266208420,"nsTiming","2"
-1,0,15,"GpuShuffleCoalesce",110,"collect batch time",359397047,"nsTiming","2"
-1,0,15,"GpuShuffleCoalesce",111,"concat batch time",104974316,"nsTiming","2"
-1,0,16,"GpuColumnarExchange",112,"partition data size",42872100,"sum","0,2"
-1,0,16,"GpuColumnarExchange",113,"partitions",200,"sum","0,2"
-1,0,16,"GpuColumnarExchange",114,"output rows",10000000,"sum","0,2"
-1,0,16,"GpuColumnarExchange",115,"output columnar batches",1200,"sum","0,2"
-1,0,16,"GpuColumnarExchange",116,"data size",40076192,"size","0,2"
-1,0,16,"GpuColumnarExchange",118,"local blocks read",1200,"sum","0,2"
-1,0,16,"GpuColumnarExchange",121,"local bytes read",40132250,"size","0,2"
-1,0,16,"GpuColumnarExchange",122,"fetch wait time",0,"timing","0,2"
-1,0,16,"GpuColumnarExchange",123,"records read",1200,"sum","0,2"
-1,0,16,"GpuColumnarExchange",124,"shuffle bytes written",40132250,"size","0,2"
-1,0,16,"GpuColumnarExchange",125,"shuffle records written",1200,"sum","0,2"
-1,0,16,"GpuColumnarExchange",126,"shuffle write time",400284505,"nsTiming","0,2"
-1,0,17,"GpuProject",127,"total time",207820,"nsTiming","0"
-1,0,18,"GpuRowToColumnar",128,"total time",58640462,"nsTiming","0"
-1,0,19,"WholeStageCodegen (2)",129,"duration",5920,"timing","0"
-1,0,21,"Scan",130,"number of output rows",10000000,"sum","0"
+appIndex,sqlID,nodeID,nodeName,accumulatorId,name,min,median,max,total,metricType,stageIds
+1,0,0,"GpuColumnarToRow",33,"total time",0,857404,857404,857404,"nsTiming","3"
+1,0,1,"GpuHashAggregate",34,"output rows",0,0,0,1,"sum","3"
+1,0,1,"GpuHashAggregate",35,"output columnar batches",0,0,0,1,"sum","3"
+1,0,1,"GpuHashAggregate",36,"total time",0,4212819,4212819,4212819,"nsTiming","3"
+1,0,1,"GpuHashAggregate",37,"aggregation time",0,3846803,3846803,3846803,"nsTiming","3"
+1,0,2,"GpuShuffleCoalesce",39,"output rows",0,0,0,200,"sum","3"
+1,0,2,"GpuShuffleCoalesce",40,"output columnar batches",0,0,0,1,"sum","3"
+1,0,2,"GpuShuffleCoalesce",41,"total time",0,3803240,3803240,3803240,"nsTiming","3"
+1,0,2,"GpuShuffleCoalesce",42,"collect batch time",0,3277904,3277904,3277904,"nsTiming","3"
+1,0,2,"GpuShuffleCoalesce",43,"concat batch time",0,392509,392509,392509,"nsTiming","3"
+1,0,3,"GpuColumnarExchange",44,"partition data size",0,0,0,16000,"sum","2,3"
+1,0,3,"GpuColumnarExchange",45,"partitions",0,0,0,1,"sum","2,3"
+1,0,3,"GpuColumnarExchange",46,"output rows",0,0,0,200,"sum","2,3"
+1,0,3,"GpuColumnarExchange",47,"output columnar batches",0,0,0,200,"sum","2,3"
+1,0,3,"GpuColumnarExchange",48,"data size",0,98,98,19600,"size","2,3"
+1,0,3,"GpuColumnarExchange",50,"local blocks read",0,0,0,200,"sum","2,3"
+1,0,3,"GpuColumnarExchange",53,"local bytes read",0,15400,15400,15400,"size","2,3"
+1,0,3,"GpuColumnarExchange",54,"fetch wait time",0,0,0,0,"timing","2,3"
+1,0,3,"GpuColumnarExchange",55,"records read",0,0,0,200,"sum","2,3"
+1,0,3,"GpuColumnarExchange",56,"shuffle bytes written",0,77,77,15400,"size","2,3"
+1,0,3,"GpuColumnarExchange",57,"shuffle records written",0,0,0,200,"sum","2,3"
+1,0,3,"GpuColumnarExchange",58,"shuffle write time",0,229727,9747416,93193331,"nsTiming","2,3"
+1,0,4,"GpuHashAggregate",59,"output rows",0,0,0,200,"sum","2"
+1,0,4,"GpuHashAggregate",60,"output columnar batches",0,0,0,200,"sum","2"
+1,0,4,"GpuHashAggregate",61,"total time",0,273719,6005323,80781515,"nsTiming","2"
+1,0,4,"GpuHashAggregate",62,"aggregation time",0,118547,1700282,31923387,"nsTiming","2"
+1,0,5,"GpuProject",64,"total time",0,15313,1360432,5377158,"nsTiming","2"
+1,0,6,"GpuShuffledHashJoin",65,"output rows",0,0,0,10000000,"sum","2"
+1,0,6,"GpuShuffledHashJoin",66,"output columnar batches",0,0,0,200,"sum","2"
+1,0,6,"GpuShuffledHashJoin",67,"total time",0,14579074,130694556,3904332009,"nsTiming","2"
+1,0,6,"GpuShuffledHashJoin",68,"build side size",0,400128,404472,80000000,"size","2"
+1,0,6,"GpuShuffledHashJoin",69,"build time",0,12797792,129063459,3448606506,"nsTiming","2"
+1,0,6,"GpuShuffledHashJoin",70,"stream time",0,925902,16602405,260796041,"nsTiming","2"
+1,0,6,"GpuShuffledHashJoin",71,"join time",0,482416,7990975,178084313,"nsTiming","2"
+1,0,6,"GpuShuffledHashJoin",72,"join output rows",0,0,0,10000000,"sum","2"
+1,0,7,"GpuShuffleCoalesce",74,"output rows",0,0,0,10000000,"sum","2"
+1,0,7,"GpuShuffleCoalesce",75,"output columnar batches",0,0,0,200,"sum","2"
+1,0,7,"GpuShuffleCoalesce",76,"total time",0,925490,16610016,261389422,"nsTiming","2"
+1,0,7,"GpuShuffleCoalesce",77,"collect batch time",0,559400,16072789,167775821,"nsTiming","2"
+1,0,7,"GpuShuffleCoalesce",78,"concat batch time",0,280723,7550015,83550919,"nsTiming","2"
+1,0,8,"GpuColumnarExchange",79,"partition data size",0,0,0,42872100,"sum","1,2"
+1,0,8,"GpuColumnarExchange",80,"partitions",0,0,0,200,"sum","1,2"
+1,0,8,"GpuColumnarExchange",81,"output rows",0,0,0,10000000,"sum","1,2"
+1,0,8,"GpuColumnarExchange",82,"output columnar batches",0,0,0,1200,"sum","1,2"
+1,0,8,"GpuColumnarExchange",83,"data size",0,6679376,6679504,40076192,"size","1,2"
+1,0,8,"GpuColumnarExchange",85,"local blocks read",0,0,0,1200,"sum","1,2"
+1,0,8,"GpuColumnarExchange",88,"local bytes read",0,200739,202927,40132258,"size","1,2"
+1,0,8,"GpuColumnarExchange",89,"fetch wait time",0,0,0,0,"timing","1,2"
+1,0,8,"GpuColumnarExchange",90,"records read",0,0,0,1200,"sum","1,2"
+1,0,8,"GpuColumnarExchange",91,"shuffle bytes written",0,6688693,6688833,40132258,"size","1,2"
+1,0,8,"GpuColumnarExchange",92,"shuffle records written",0,0,0,1200,"sum","1,2"
+1,0,8,"GpuColumnarExchange",93,"shuffle write time",0,85999094,108992798,508750471,"nsTiming","1,2"
+1,0,9,"GpuProject",94,"total time",0,35976,6485908,6667140,"nsTiming","1"
+1,0,10,"GpuRowToColumnar",95,"total time",0,8978666,16869329,61112304,"nsTiming","1"
+1,0,11,"WholeStageCodegen (1)",96,"duration",0,900,1048,5463,"timing","1"
+1,0,13,"Scan",97,"number of output rows",0,0,0,10000000,"sum","1"
+1,0,14,"GpuCoalesceBatches",98,"output rows",0,0,0,10000000,"sum","2"
+1,0,14,"GpuCoalesceBatches",99,"output columnar batches",0,0,0,200,"sum","2"
+1,0,14,"GpuCoalesceBatches",100,"total time",0,12667478,127292703,3383354389,"nsTiming","2"
+1,0,14,"GpuCoalesceBatches",101,"collect batch time",0,12358455,126548235,3275108263,"nsTiming","2"
+1,0,14,"GpuCoalesceBatches",102,"concat batch time",0,61806,2606918,20312708,"nsTiming","2"
+1,0,14,"GpuCoalesceBatches",103,"peak device memory",0,400128,404472,80000000,"size","2"
+1,0,15,"GpuShuffleCoalesce",107,"output rows",0,0,0,10000000,"sum","2"
+1,0,15,"GpuShuffleCoalesce",108,"output columnar batches",0,0,0,200,"sum","2"
+1,0,15,"GpuShuffleCoalesce",109,"total time",0,12348751,124764640,3266208420,"nsTiming","2"
+1,0,15,"GpuShuffleCoalesce",110,"collect batch time",0,554106,21274218,359397047,"nsTiming","2"
+1,0,15,"GpuShuffleCoalesce",111,"concat batch time",0,301378,18212719,104974316,"nsTiming","2"
+1,0,16,"GpuColumnarExchange",112,"partition data size",0,0,0,42872100,"sum","0,2"
+1,0,16,"GpuColumnarExchange",113,"partitions",0,0,0,200,"sum","0,2"
+1,0,16,"GpuColumnarExchange",114,"output rows",0,0,0,10000000,"sum","0,2"
+1,0,16,"GpuColumnarExchange",115,"output columnar batches",0,0,0,1200,"sum","0,2"
+1,0,16,"GpuColumnarExchange",116,"data size",0,6679376,6679504,40076192,"size","0,2"
+1,0,16,"GpuColumnarExchange",118,"local blocks read",0,0,0,1200,"sum","0,2"
+1,0,16,"GpuColumnarExchange",121,"local bytes read",0,200739,202927,40132250,"size","0,2"
+1,0,16,"GpuColumnarExchange",122,"fetch wait time",0,0,0,0,"timing","0,2"
+1,0,16,"GpuColumnarExchange",123,"records read",0,0,0,1200,"sum","0,2"
+1,0,16,"GpuColumnarExchange",124,"shuffle bytes written",0,6688678,6688825,40132250,"size","0,2"
+1,0,16,"GpuColumnarExchange",125,"shuffle records written",0,0,0,1200,"sum","0,2"
+1,0,16,"GpuColumnarExchange",126,"shuffle write time",0,51600615,100858775,400284505,"nsTiming","0,2"
+1,0,17,"GpuProject",127,"total time",0,33979,41588,207820,"nsTiming","0"
+1,0,18,"GpuRowToColumnar",128,"total time",0,8472944,20415869,58640462,"nsTiming","0"
+1,0,19,"WholeStageCodegen (2)",129,"duration",0,977,1092,5920,"timing","0"
+1,0,21,"Scan",130,"number of output rows",0,0,0,10000000,"sum","0"