diff --git a/common/kvstore/src/main/java/org/apache/spark/kvstore/LevelDBIterator.java b/common/kvstore/src/main/java/org/apache/spark/kvstore/LevelDBIterator.java
index 8b0a9344aa92c..0a4b29d2453fb 100644
--- a/common/kvstore/src/main/java/org/apache/spark/kvstore/LevelDBIterator.java
+++ b/common/kvstore/src/main/java/org/apache/spark/kvstore/LevelDBIterator.java
@@ -89,7 +89,16 @@ class LevelDBIterator<T> implements KVStoreIterator<T> {
     } else {
       this.end = null;
       if (it.hasNext()) {
-        it.next();
+        // When descending, the caller may have set up the start of iteration at a non-existant
+        // entry that is guaranteed to be after the desired entry. For example, if you have a
+        // compound key (a, b) where b is a, integer, you may seek to the end of the elements that
+        // have the same "a" value by specifying Integer.MAX_VALUE for "b", and that value may not
+        // exist in the database. So need to check here whether the next value actually belongs to
+        // the set being returned by the iterator before advancing.
+        byte[] nextKey = it.peekNext().getKey();
+        if (compare(nextKey, indexKeyPrefix) <= 0) {
+          it.next();
+        }
       }
     }
 
@@ -258,4 +267,17 @@ private byte[] stitch(byte[]... comps) {
     return dest;
   }
 
+  private int compare(byte[] a, byte[] b) {
+    int diff = 0;
+    int minLen = Math.min(a.length, b.length);
+    for (int i = 0; i < minLen; i++) {
+      diff += (a[i] - b[i]);
+      if (diff != 0) {
+        return diff;
+      }
+    }
+
+    return a.length - b.length;
+  }
+
 }
diff --git a/core/src/main/java/org/apache/spark/status/api/v1/StageStatus.java b/core/src/main/java/org/apache/spark/status/api/v1/StageStatus.java
index 9dbb565aab707..40b5f627369d5 100644
--- a/core/src/main/java/org/apache/spark/status/api/v1/StageStatus.java
+++ b/core/src/main/java/org/apache/spark/status/api/v1/StageStatus.java
@@ -23,7 +23,8 @@ public enum StageStatus {
   ACTIVE,
   COMPLETE,
   FAILED,
-  PENDING;
+  PENDING,
+  SKIPPED;
 
   public static StageStatus fromString(String str) {
     return EnumUtil.parseIgnoreCase(StageStatus.class, str);
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 9286dde341a44..527b45ea102d1 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -455,7 +455,7 @@ class SparkContext(config: SparkConf) extends Logging {
 
     _ui =
       if (conf.getBoolean("spark.ui.enabled", true)) {
-        Some(SparkUI.create(Some(this), _stateStore, _conf, listenerBus, _env.securityManager,
+        Some(SparkUI.create(Some(this), _stateStore, _conf, _env.securityManager,
           appName, "", startTime))
       } else {
         // For tests, do not enable the UI
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index 01fc2291d6f13..01968918909c1 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -284,12 +284,6 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       return None
     }
 
-    val path = uiStorePath(appId, attemptId)
-    if (!path.exists()) {
-      throw new IllegalStateException(
-        s"Application entry for $appId / $attemptId found, but UI not available.")
-    }
-
     val conf = this.conf.clone()
     val secManager = new SecurityManager(conf)
 
@@ -301,13 +295,18 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
       attempt.adminAclsGroups.getOrElse(""))
     secManager.setViewAclsGroups(attempt.viewAclsGroups.getOrElse(""))
 
+    val path = uiStorePath(appId, attemptId)
+    if (!path.exists()) {
+      throw new IllegalStateException(
+        s"Application entry for $appId / $attemptId found, but UI not available.")
+    }
+
     val replayBus = new ReplayListenerBus()
 
     // Create the UI under a lock so that a valid disk store is used, in case the update thread
     // is writing a new disk store for the application (see replaceStore()).
     val loadedUI = synchronized {
-      val ui = SparkUI.create(None, AppStateStore.loadStore(conf, path), conf, replayBus,
-        secManager,
+      val ui = SparkUI.create(None, AppStateStore.loadStore(conf, path), conf, secManager,
         app.info.name,
         HistoryServer.getAttemptURI(appId, attempt.info.attemptId),
         attempt.info.startTime.getTime())
diff --git a/core/src/main/scala/org/apache/spark/status/AppStateListener.scala b/core/src/main/scala/org/apache/spark/status/AppStateListener.scala
index 5d875757e3736..883eccbcef904 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStateListener.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStateListener.scala
@@ -31,18 +31,24 @@ import org.apache.spark.scheduler._
 import org.apache.spark.status.api.v1
 import org.apache.spark.storage._
 import org.apache.spark.ui.SparkUI
+import org.apache.spark.ui.scope._
+import org.apache.spark.util.AccumulatorContext
 
 /**
  * A Spark listener that writes application information to a data store. The types written to the
  * store are defined in the `storeTypes.scala` file and are based on the public REST API.
  *
  * TODO (M4):
- * - Need to add state / information needed by the UI that is currently not in the API.
  * - Cache active jobs / stages / other interesting things in memory to make it faster
  *   to update them
  * - Flush data to the store in a separate thread (to avoid stalling the listener bus).
  *
  * TODO (future):
+ * - Add support for cleaning up old data so that the store doesn't grow arbitrarily. This is
+ *   especially important for providing an in-memory store for when the JNI libraries are not
+ *   available.
+ *
+ * TODO (future):
  * - to enable incremental parsing of event logs, all state in this class needs to be serialized
  *   to the underlying store and loaded when the class is instantiated. That could potentially
  *   be written to the AppStatusStoreMetadata object.
@@ -204,16 +210,14 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       missingStages.map(_.numTasks).sum
     }
 
-    val lastStageInfo = event.stageInfos.lastOption
+    val lastStageInfo = event.stageInfos.sortBy(_.stageId).lastOption
     val lastStageName = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
-
     val jobGroup = Option(event.properties)
       .flatMap { p => Option(p.getProperty(SparkContext.SPARK_JOB_GROUP_ID)) }
 
     val info = newJobData(None,
       jobId = event.jobId,
       name = lastStageName,
-      description = lastStageInfo.map(_.details),
       submissionTime = Option(event.time).filter(_ >= 0).map(new Date(_)),
       stageIds = event.stageIds,
       jobGroup = jobGroup,
@@ -225,15 +229,40 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       // instead of just a write.
       updateStageData(stage.stageId, stage.attemptId) { wrapper =>
         val jobIds = wrapper.jobIds ++ Set(event.jobId)
-        newStageDataWrapper(wrapper, wrapper.info, jobIds = jobIds)
+        val newStage = newStageData(
+          wrapper.info,
+          name = stage.name,
+          numTasks = stage.numTasks,
+          rddIds = stage.rddInfos.map(_.id))
+        newStageDataWrapper(wrapper, newStage, jobIds = jobIds)
       }
-      (stage.stageId, stage.attemptId)
-    }.toSet
+      new JobStageSummary(stage.stageId, stage.attemptId, stage.numTasks)
+    }
 
     val stored = newJobDataWrapper(None, info, initialStages = initialStages)
     kvstore.write(stored)
 
     activeJobs += event.jobId
+
+    // Create the graph data for all the job's stages.
+    event.stageInfos.foreach { stage =>
+      val graph = RDDOperationGraph.makeOperationGraph(stage, Int.MaxValue)
+      val uigraph = new RDDOperationGraphWrapper(
+        stage.stageId,
+        graph.edges,
+        graph.outgoingEdges,
+        graph.incomingEdges,
+        newRDDOperationCluster(graph.rootCluster))
+      kvstore.write(uigraph)
+    }
+  }
+
+  private def newRDDOperationCluster(cluster: RDDOperationCluster): RDDOperationClusterWrapper = {
+    new RDDOperationClusterWrapper(
+      cluster.id,
+      cluster.name,
+      cluster.childNodes,
+      cluster.childClusters.map(newRDDOperationCluster))
   }
 
   override def onJobEnd(event: SparkListenerJobEnd): Unit = {
@@ -243,23 +272,22 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         case JobFailed(_) => JobExecutionStatus.FAILED
       }
 
-      val skippedStages = job.info.stageIds.toSet &~ job.completedStages
+      val skippedStageIds = job.info.stageIds.toSet &~ job.submittedStages
+      val skippedStages = job.initialStages.filter { s => skippedStageIds.contains(s.stageId) }
 
-      // Calculate the number of skipped tasks based on the number of completed tasks of
-      // skipped stage attempts.
-      val skippedAttempts = job.initialStages.filter { case (stageId, _) =>
-        skippedStages.contains(stageId)
+      skippedStages.foreach { s =>
+        updateStageData(s.stageId, s.attemptId) { stage =>
+          val newInfo = newStageData(stage.info, status = v1.StageStatus.SKIPPED)
+          newStageDataWrapper(stage, newInfo)
+        }
       }
-      val skippedTasks = skippedAttempts.flatMap { case (id, attempt) =>
-        read[StageDataWrapper](Array(id, attempt)).map(_.info.numCompleteTasks)
-      }.sum
 
       val newInfo = newJobData(job.info,
         jobId = event.jobId,
-        completionTime = Some(new Date(event.time)),
+        completionTime = if (event.time != -1) Some(new Date(event.time)) else None,
         status = status,
         numSkippedStages = skippedStages.size,
-        numSkippedTasks = skippedTasks)
+        numSkippedTasks = skippedStages.map(_.numTasks).sum)
       newJobDataWrapper(job, newInfo)
     }
 
@@ -276,23 +304,32 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       event.stageInfo.attemptId
     }
 
-    updateStageData(event.stageInfo.stageId, previousAttempt) { stage =>
-      val poolName = Option(event.properties).flatMap { p =>
-        Option(p.getProperty("spark.scheduler.pool"))
-      }.getOrElse(SparkUI.DEFAULT_POOL_NAME)
+    val poolName = Option(event.properties).flatMap { p =>
+      Option(p.getProperty("spark.scheduler.pool"))
+    }.getOrElse(SparkUI.DEFAULT_POOL_NAME)
 
+    updateStageData(event.stageInfo.stageId, previousAttempt) { stage =>
       val description = Option(event.properties).flatMap { p =>
         Option(p.getProperty(SparkContext.SPARK_JOB_DESCRIPTION))
       }
 
-      // Update jobs to add active stage and remove the stage from the completed list.
+      // Update the jobs related to the stage.
+      // - update the stage counts
+      // - remove the stage from the job's completed list
+      // - add the stage to the list of submitted stages for the stage, so that skipped stages
+      //   are correctly counted.
       activeJobs(stage).foreach { job =>
         updateJobData(job) { wrapper =>
+          val completedStages = wrapper.completedStages - event.stageInfo.stageId
+          val submittedStages = wrapper.submittedStages + event.stageInfo.stageId
           val newJobInfo = newJobData(
             wrapper.info,
             jobId = job,
-            numActiveStages = wrapper.info.numActiveStages + 1)
-          newJobDataWrapper(wrapper, newJobInfo)
+            numActiveStages = wrapper.info.numActiveStages + 1,
+            numCompletedStages = completedStages.size)
+          newJobDataWrapper(wrapper, newJobInfo,
+            submittedStages = submittedStages,
+            completedStages = completedStages)
         }
       }
 
@@ -301,12 +338,19 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         status = v1.StageStatus.ACTIVE,
         stageId = event.stageInfo.stageId,
         attemptId = event.stageInfo.attemptId,
+        numTasks = event.stageInfo.numTasks,
         submissionTime = event.stageInfo.submissionTime.map(new Date(_)),
-        details = description,
+        name = event.stageInfo.name,
+        description = description,
+        details = event.stageInfo.details,
         schedulingPool = poolName)
       newStageDataWrapper(stage, newInfo)
     }
 
+    updatePoolData(poolName) { pool =>
+      new PoolData(pool.name, pool.stageIds + event.stageInfo.stageId)
+    }
+
     // Update RDDs with the stage name.
     event.stageInfo.rddInfos.foreach { info =>
       updateRDDStorageInfo(info.id) { rdd =>
@@ -358,17 +402,23 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       newStageDataWrapper(stage, newInfo)
     }
 
-    updateExecutorSummary(event.taskInfo.executorId) { uiexec =>
+    updateExecutorSummary(event.taskInfo.executorId) { exec =>
       val updated = newExecutorSummary(
-        uiexec.info,
-        activeTasks = uiexec.info.activeTasks + 1,
-        totalTasks = uiexec.info.totalTasks + 1)
+        exec.info,
+        activeTasks = exec.info.activeTasks + 1,
+        totalTasks = exec.info.totalTasks + 1)
       new ExecutorSummaryWrapper(updated)
     }
   }
 
   override def onTaskGettingResult(event: SparkListenerTaskGettingResult): Unit = {
-    // Nothing to do here.
+    updateTaskData(event.taskInfo.taskId) { task =>
+      val newInfo = newTaskData(
+        task.info,
+        status = event.taskInfo.status,
+        resultFetchStart = Some(new Date(event.taskInfo.gettingResultTime)))
+      new TaskDataWrapper(newInfo, task.stageId, task.stageAttemptId)
+    }
   }
 
   override def onTaskEnd(event: SparkListenerTaskEnd): Unit = {
@@ -382,6 +432,12 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
 
     var metricsDelta: v1.TaskMetrics = null
 
+    val executorRunTime = if (event.taskMetrics != null) {
+      event.taskMetrics.executorRunTime
+    } else {
+      event.taskInfo.duration
+    }
+
     updateTaskData(event.taskInfo.taskId) { task =>
       val errorMessage = event.reason match {
         case Success =>
@@ -401,22 +457,26 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         taskId = event.taskInfo.taskId,
         duration = Some(event.taskInfo.duration),
         status = event.taskInfo.status,
+        accumulatorUpdates = filterAccumulables(info.accumulables).toSeq.map(newAccumulableInfo),
         errorMessage = errorMessage,
         taskMetrics = Option(event.taskMetrics).map(newTaskMetrics))
       new TaskDataWrapper(newInfo, event.stageId, event.stageAttemptId)
     }
 
-    val (completedDelta, failedDelta) = event.reason match {
+    val (completedDelta, failedDelta, killedDelta) = event.reason match {
       case Success =>
-        (1, 0)
+        (1, 0, 0)
       case _: TaskKilled =>
-        (0, 0)
+        (0, 0, 1)
       case _ =>
-        (0, 1)
+        (0, 1, 0)
     }
 
     updateStageData(event.stageId, event.stageAttemptId) { stage =>
-      val accumulables = info.accumulables.map(newAccumulableInfo).toSeq
+      val killReason = event.reason match {
+        case k: TaskKilled => Some(k.reason)
+        case _ => None
+      }
 
       val oldStage = stage.info
       val newInfo = newStageData(
@@ -424,7 +484,8 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         numActiveTasks = oldStage.numActiveTasks - 1,
         numCompleteTasks = oldStage.numCompleteTasks + completedDelta,
         numFailedTasks = oldStage.numFailedTasks + failedDelta,
-        accumulatorUpdates = accumulables,
+        numKilledTasks = oldStage.numKilledTasks + killedDelta,
+        executorRunTime = oldStage.executorRunTime + executorRunTime,
         inputBytes = oldStage.inputBytes + metricsDelta.inputMetrics.bytesRead,
         inputRecords = oldStage.inputRecords + metricsDelta.inputMetrics.recordsRead,
         outputBytes = oldStage.outputBytes + metricsDelta.outputMetrics.bytesWritten,
@@ -439,7 +500,9 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         shuffleWriteRecords = oldStage.shuffleWriteRecords +
           metricsDelta.shuffleWriteMetrics.recordsWritten,
         memoryBytesSpilled = oldStage.memoryBytesSpilled + metricsDelta.memoryBytesSpilled,
-        diskBytesSpilled = oldStage.diskBytesSpilled + metricsDelta.diskBytesSpilled)
+        diskBytesSpilled = oldStage.diskBytesSpilled + metricsDelta.diskBytesSpilled,
+        accumulatorUpdates = mergeAccumulables(oldStage.accumulatorUpdates, info.accumulables),
+        killedTasksSummary = killedTasksSummary(event.reason, oldStage.killedTasksSummary))
 
       activeJobs(stage).foreach { job =>
         updateJobData(job) { wrapper =>
@@ -448,7 +511,9 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
             jobId = job,
             numActiveTasks = wrapper.info.numActiveTasks - 1,
             numCompletedTasks = wrapper.info.numCompletedTasks + completedDelta,
-            numFailedTasks = wrapper.info.numFailedTasks + failedDelta)
+            numFailedTasks = wrapper.info.numFailedTasks + failedDelta,
+            numKilledTasks = wrapper.info.numKilledTasks + killedDelta,
+            killedTasksSummary = killedTasksSummary(event.reason, wrapper.info.killedTasksSummary))
           newJobDataWrapper(wrapper, newJobInfo)
         }
       }
@@ -462,18 +527,22 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         event.taskInfo.executorId) { exec =>
       newExecutorStageSummary(
         exec,
+        taskTime = exec.taskTime + info.duration,
         failedTasks = exec.failedTasks + failedDelta,
         succeededTasks = exec.succeededTasks + completedDelta,
         inputBytes = exec.inputBytes + metricsDelta.inputMetrics.bytesRead,
         outputBytes = exec.outputBytes + metricsDelta.outputMetrics.bytesWritten,
         shuffleRead = exec.shuffleRead + metricsDelta.shuffleReadMetrics.localBytesRead +
           metricsDelta.shuffleReadMetrics.remoteBytesRead,
+        shuffleReadRecords = exec.shuffleReadRecords + metricsDelta.shuffleReadMetrics.recordsRead,
         shuffleWrite = exec.shuffleWrite + metricsDelta.shuffleWriteMetrics.bytesWritten,
+        shuffleWriteRecords = exec.shuffleWriteRecords +
+          metricsDelta.shuffleWriteMetrics.recordsWritten,
         memoryBytesSpilled = exec.memoryBytesSpilled + metricsDelta.memoryBytesSpilled,
         diskBytesSpilled = exec.diskBytesSpilled + metricsDelta.diskBytesSpilled)
     }
 
-    updateExecutorSummary(event.taskInfo.executorId) { uiexec =>
+    updateExecutorSummary(event.taskInfo.executorId) { exec =>
       val (gcTime, inputBytes, shuffleRead, shuffleWrite) = if (event.taskMetrics != null) {
         val readMetrics = event.taskMetrics.shuffleReadMetrics
         (event.taskMetrics.jvmGCTime,
@@ -485,15 +554,15 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       }
 
       val updated = newExecutorSummary(
-        uiexec.info,
-        activeTasks = uiexec.info.activeTasks - 1,
-        completedTasks = uiexec.info.completedTasks + completedDelta,
-        failedTasks = uiexec.info.failedTasks + failedDelta,
-        totalDuration = uiexec.info.totalDuration + event.taskInfo.duration,
-        totalGCTime = uiexec.info.totalGCTime + gcTime,
-        totalInputBytes = uiexec.info.totalInputBytes + inputBytes,
-        totalShuffleRead = uiexec.info.totalShuffleRead + shuffleRead,
-        totalShuffleWrite = uiexec.info.totalShuffleWrite + shuffleWrite)
+        exec.info,
+        activeTasks = exec.info.activeTasks - 1,
+        completedTasks = exec.info.completedTasks + completedDelta,
+        failedTasks = exec.info.failedTasks + failedDelta,
+        totalDuration = exec.info.totalDuration + event.taskInfo.duration,
+        totalGCTime = exec.info.totalGCTime + gcTime,
+        totalInputBytes = exec.info.totalInputBytes + inputBytes,
+        totalShuffleRead = exec.info.totalShuffleRead + shuffleRead,
+        totalShuffleWrite = exec.info.totalShuffleWrite + shuffleWrite)
       new ExecutorSummaryWrapper(updated)
     }
   }
@@ -505,10 +574,19 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
         case Some(_) => v1.StageStatus.FAILED
         case None => v1.StageStatus.COMPLETE
       }
+
+      // For some reason this data is filled in the "completed" event, not in the "submitted"
+      // event. Go figure.
+      val submissionTime = event.stageInfo.submissionTime.map(new Date(_)).orElse(
+        stage.info.submissionTime)
+      val accumulables = filterAccumulables(info.accumulables.values).map(newAccumulableInfo).toSeq
       val newStage = newStageData(
         stage.info,
         status = status,
-        completionTime = info.completionTime.map(new Date(_)))
+        submissionTime = submissionTime,
+        completionTime = info.completionTime.map(new Date(_)),
+        failureReason = info.failureReason,
+        accumulatorUpdates = accumulables)
 
       val (completedDelta, failedDelta) = if (status == v1.StageStatus.COMPLETE) {
         (1, 0)
@@ -518,17 +596,25 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
 
       activeJobs(stage).foreach { job =>
         updateJobData(job) { wrapper =>
+          val stages = if (status == v1.StageStatus.COMPLETE) {
+            wrapper.completedStages + info.stageId
+          } else {
+            wrapper.completedStages
+          }
           val newJobInfo = newJobData(
             wrapper.info,
             jobId = job,
             numActiveStages = wrapper.info.numActiveStages - 1,
-            numCompletedStages = wrapper.info.numCompletedStages + completedDelta,
+            numCompletedStages = stages.size,
             numFailedStages = wrapper.info.numFailedStages + failedDelta)
-          val stages = wrapper.completedStages + info.stageId
           newJobDataWrapper(wrapper, newJobInfo, completedStages = stages)
         }
       }
 
+      updatePoolData(newStage.schedulingPool) { pool =>
+        new PoolData(pool.name, pool.stageIds - newStage.stageId)
+      }
+
       newStageDataWrapper(stage, newStage)
     }
   }
@@ -572,6 +658,11 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
 
       updateStageData(sid, sAttempt) { stage =>
         val oldStage = stage.info
+
+        // Note that this update does not update the stage's accumulator list. The updates
+        // received in the event are partial and only refer to a specific executor, so there
+        // isn't enough information to update the non-internal accumulators with their current
+        // values.
         val updatedStage = newStageData(
           oldStage,
           inputBytes = oldStage.inputBytes + delta.inputMetrics.bytesRead,
@@ -599,7 +690,9 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
           outputBytes = exec.outputBytes + delta.outputMetrics.bytesWritten,
           shuffleRead = exec.shuffleRead + delta.shuffleReadMetrics.localBytesRead +
             delta.shuffleReadMetrics.remoteBytesRead,
+          shuffleReadRecords = exec.shuffleReadRecords + delta.shuffleReadMetrics.recordsRead,
           shuffleWrite = exec.shuffleWrite + delta.shuffleWriteMetrics.bytesWritten,
+          shuffleWriteRecords = exec.shuffleWriteRecords + delta.shuffleWriteMetrics.recordsWritten,
           memoryBytesSpilled = exec.memoryBytesSpilled + delta.memoryBytesSpilled,
           diskBytesSpilled = exec.diskBytesSpilled + delta.diskBytesSpilled)
       }
@@ -853,6 +946,12 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
     }
   }
 
+  private def updatePoolData(name: String)(fn: PoolData => PoolData): Unit = {
+    update[PoolData](name) { old =>
+      fn(old.getOrElse(new PoolData(name, Set())))
+    }
+  }
+
   /**
    * Return a new TaskMetrics object containing the delta of the various fields of the given
    * metrics objects. This is currently targeted at updating stage data, so it does not
@@ -891,6 +990,7 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       0L, 0L, 0L,
       metrics.memoryBytesSpilled - old.memoryBytesSpilled,
       metrics.diskBytesSpilled - old.diskBytesSpilled,
+      0L,
       inputDelta,
       outputDelta,
       shuffleReadDelta,
@@ -978,10 +1078,12 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       numCompletedTasks: Option[Int] = None,
       numSkippedTasks: Option[Int] = None,
       numFailedTasks: Option[Int] = None,
+      numKilledTasks: Option[Int] = None,
       numActiveStages: Option[Int] = None,
       numCompletedStages: Option[Int] = None,
       numSkippedStages: Option[Int] = None,
-      numFailedStages: Option[Int] = None): v1.JobData = {
+      numFailedStages: Option[Int] = None,
+      killedTasksSummary: Option[Map[String, Int]] = None): v1.JobData = {
     new v1.JobData(
       value(jobId, old.map(_.jobId), -1),
       value(name, old.map(_.name), null),
@@ -996,10 +1098,12 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       value(numCompletedTasks, old.map(_.numCompletedTasks), 0),
       value(numSkippedTasks, old.map(_.numSkippedTasks), 0),
       value(numFailedTasks, old.map(_.numFailedTasks), 0),
+      value(numKilledTasks, old.map(_.numKilledTasks), 0),
       value(numActiveStages, old.map(_.numActiveStages), 0),
       value(numCompletedStages, old.map(_.numCompletedStages), 0),
       value(numSkippedStages, old.map(_.numSkippedStages), 0),
-      value(numFailedStages, old.map(_.numFailedStages), 0))
+      value(numFailedStages, old.map(_.numFailedStages), 0),
+      value(killedTasksSummary, old.map(_.killedTasksSummary), Map()))
   }
 
   private def newStageData(
@@ -1007,14 +1111,17 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       status: Option[v1.StageStatus] = None,
       stageId: Option[Int] = None,
       attemptId: Option[Int] = None,
+      numTasks: Option[Int] = None,
       numActiveTasks: Option[Int] = None,
       numCompleteTasks: Option[Int] = None,
       numFailedTasks: Option[Int] = None,
+      numKilledTasks: Option[Int] = None,
       executorRunTime: Option[Long] = None,
       executorCpuTime: Option[Long] = None,
       submissionTime: Option[Option[Date]] = None,
       firstTaskLaunchedTime: Option[Option[Date]] = None,
       completionTime: Option[Option[Date]] = None,
+      failureReason: Option[Option[String]] = None,
       inputBytes: Option[Long] = None,
       inputRecords: Option[Long] = None,
       outputBytes: Option[Long] = None,
@@ -1026,22 +1133,28 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       memoryBytesSpilled: Option[Long] = None,
       diskBytesSpilled: Option[Long] = None,
       name: Option[String] = None,
+      description: Option[Option[String]] = None,
       details: Option[String] = None,
       schedulingPool: Option[String] = None,
-      accumulatorUpdates: Option[Seq[v1.AccumulableInfo]] = None)
+      rddIds: Option[Seq[Int]] = None,
+      accumulatorUpdates: Option[Seq[v1.AccumulableInfo]] = None,
+      killedTasksSummary: Option[Map[String, Int]] = None)
   : v1.StageData = {
     new v1.StageData(
       value(status, old.map(_.status), v1.StageStatus.PENDING),
       value(stageId, old.map(_.stageId), -1),
       value(attemptId, old.map(_.attemptId), -1),
+      value(numTasks, old.map(_.numTasks), 0),
       value(numActiveTasks, old.map(_.numActiveTasks), 0),
       value(numCompleteTasks, old.map(_.numCompleteTasks), 0),
       value(numFailedTasks, old.map(_.numFailedTasks), 0),
+      value(numKilledTasks, old.map(_.numKilledTasks), 0),
       value(executorRunTime, old.map(_.executorRunTime), 0L),
       value(executorCpuTime, old.map(_.executorCpuTime), 0L),
       option(submissionTime, old.map(_.submissionTime)),
       option(firstTaskLaunchedTime, old.map(_.firstTaskLaunchedTime)),
       option(completionTime, old.map(_.completionTime)),
+      option(failureReason, old.map(_.failureReason)),
       value(inputBytes, old.map(_.inputBytes), 0L),
       value(inputRecords, old.map(_.inputRecords), 0L),
       value(outputBytes, old.map(_.outputBytes), 0L),
@@ -1053,11 +1166,14 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       value(memoryBytesSpilled, old.map(_.memoryBytesSpilled), 0L),
       value(diskBytesSpilled, old.map(_.diskBytesSpilled), 0L),
       value(name, old.map(_.name), null),
+      option(description, old.map(_.description)),
       value(details, old.map(_.details), null),
       value(schedulingPool, old.map(_.schedulingPool), null),
+      value(rddIds, old.map(_.rddIds), Nil),
       value(accumulatorUpdates, old.map(_.accumulatorUpdates), Nil),
       None, // Task list is always empty; it's stored separately.
-      None) // Executor summary us always empty; it's stored separately.
+      None, // Executor summary us always empty; it's stored separately.
+      value(killedTasksSummary, old.map(_.killedTasksSummary), Map()))
   }
 
   private def newExecutorStageSummary(
@@ -1065,20 +1181,30 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       taskTime: Option[Long] = None,
       failedTasks: Option[Int] = None,
       succeededTasks: Option[Int] = None,
+      killedTasks: Option[Int] = None,
       inputBytes: Option[Long] = None,
+      inputRecords: Option[Long] = None,
       outputBytes: Option[Long] = None,
+      outputRecords: Option[Long] = None,
       shuffleRead: Option[Long] = None,
+      shuffleReadRecords: Option[Long] = None,
       shuffleWrite: Option[Long] = None,
+      shuffleWriteRecords: Option[Long] = None,
       memoryBytesSpilled: Option[Long] = None,
       diskBytesSpilled: Option[Long] = None): v1.ExecutorStageSummary = {
     new v1.ExecutorStageSummary(
       value(taskTime, old.map(_.taskTime), 0L),
       value(failedTasks, old.map(_.failedTasks), 0),
       value(succeededTasks, old.map(_.succeededTasks), 0),
+      value(killedTasks, old.map(_.killedTasks), 0),
       value(inputBytes, old.map(_.inputBytes), 0L),
+      value(inputRecords, old.map(_.inputRecords), 0L),
       value(outputBytes, old.map(_.outputBytes), 0L),
+      value(outputRecords, old.map(_.outputRecords), 0L),
       value(shuffleRead, old.map(_.shuffleRead), 0L),
+      value(shuffleReadRecords, old.map(_.shuffleReadRecords), 0L),
       value(shuffleWrite, old.map(_.shuffleWrite), 0L),
+      value(shuffleWriteRecords, old.map(_.shuffleWriteRecords), 0L),
       value(memoryBytesSpilled, old.map(_.memoryBytesSpilled), 0L),
       value(diskBytesSpilled, old.map(_.diskBytesSpilled), 0L))
   }
@@ -1089,6 +1215,7 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       index: Option[Int] = None,
       attempt: Option[Int] = None,
       launchTime: Option[Date] = None,
+      resultFetchStart: Option[Option[Date]] = None,
       duration: Option[Option[Long]] = None,
       executorId: Option[String] = None,
       host: Option[String] = None,
@@ -1103,11 +1230,12 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       value(index, old.map(_.index), -1),
       value(attempt, old.map(_.attempt), -1),
       value(launchTime, old.map(_.launchTime), DEFAULT_DATE),
+      option(resultFetchStart, old.map(_.resultFetchStart)),
       option(duration, old.map(_.duration)),
-      value(executorId, old.map(_.executorId), UNKNOWN),
-      value(host, old.map(_.host), UNKNOWN),
-      value(status, old.map(_.status), UNKNOWN),
-      value(taskLocality, old.map(_.taskLocality), UNKNOWN),
+      value(executorId, old.map(_.executorId), UNKNOWN).intern(),
+      value(host, old.map(_.host), UNKNOWN).intern(),
+      value(status, old.map(_.status), UNKNOWN).intern(),
+      value(taskLocality, old.map(_.taskLocality), UNKNOWN).intern(),
       value(speculative, old.map(_.speculative), false),
       value(accumulatorUpdates, old.map(_.accumulatorUpdates), Nil),
       option(errorMessage, old.map(_.errorMessage)),
@@ -1178,7 +1306,7 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
   private def newAccumulableInfo(acc: AccumulableInfo): v1.AccumulableInfo = {
     new v1.AccumulableInfo(
       acc.id,
-      acc.name.orNull,
+      acc.name.map(_.intern()).orNull,
       acc.update.map(_.toString()),
       acc.value.map(_.toString()).orNull)
   }
@@ -1194,6 +1322,7 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
       metrics.resultSerializationTime,
       metrics.memoryBytesSpilled,
       metrics.diskBytesSpilled,
+      metrics.peakExecutionMemory,
       new v1.InputMetrics(
         metrics.inputMetrics.bytesRead,
         metrics.inputMetrics.recordsRead),
@@ -1216,10 +1345,12 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
   private def newJobDataWrapper(
       old: Option[JobDataWrapper],
       info: v1.JobData,
-      initialStages: Option[Set[(Int, Int)]] = None,
+      initialStages: Option[Seq[JobStageSummary]] = None,
+      submittedStages: Option[Set[Int]] = None,
       completedStages: Option[Set[Int]] = None): JobDataWrapper = {
     new JobDataWrapper(info,
-      value(initialStages, old.map(_.initialStages), Set()),
+      value(initialStages, old.map(_.initialStages), Seq()),
+      value(submittedStages, old.map(_.submittedStages), Set()),
       value(completedStages, old.map(_.completedStages), Set()))
   }
 
@@ -1234,6 +1365,37 @@ private class AppStateListener(override protected val kvstore: KVStore) extends
     stage.jobIds.filter(activeJobs.contains(_))
   }
 
+  private def filterAccumulables(accums: Iterable[AccumulableInfo]): Iterable[AccumulableInfo] = {
+    accums.filter { acc =>
+      // We don't need to store internal or SQL accumulables as their values will be shown in
+      // other places, so drop them to reduce the memory usage.
+      !acc.internal && acc.metadata != Some(AccumulatorContext.SQL_ACCUM_IDENTIFIER)
+    }
+  }
+
+  private def mergeAccumulables(
+      old: Seq[v1.AccumulableInfo],
+      updates: Iterable[AccumulableInfo]): Seq[v1.AccumulableInfo] = {
+    val newAccums = filterAccumulables(updates)
+      .map { acc =>
+        acc.id -> newAccumulableInfo(acc)
+      }.toMap
+
+    val oldAccums = old.map { acc => acc.id -> acc }.toMap
+    (oldAccums ++ newAccums).values.toSeq.sortBy(_.id)
+  }
+
+  private def killedTasksSummary(
+      reason: TaskEndReason,
+      oldSummary: Map[String, Int]): Map[String, Int] = {
+    reason match {
+      case k: TaskKilled =>
+        oldSummary.updated(k.reason, oldSummary.getOrElse(k.reason, 0) + 1)
+      case _ =>
+        oldSummary
+    }
+  }
+
 }
 
 private[spark] object AppStateListener {
diff --git a/core/src/main/scala/org/apache/spark/status/AppStateStore.scala b/core/src/main/scala/org/apache/spark/status/AppStateStore.scala
index 9db54f6345f2a..36dc2979aa02a 100644
--- a/core/src/main/scala/org/apache/spark/status/AppStateStore.scala
+++ b/core/src/main/scala/org/apache/spark/status/AppStateStore.scala
@@ -23,9 +23,10 @@ import java.util.{Arrays, List => JList}
 import scala.collection.JavaConverters._
 
 import org.apache.spark.{JobExecutionStatus, SparkConf}
-import org.apache.spark.kvstore.KVStore
+import org.apache.spark.kvstore.{KVStore, KVStoreView}
 import org.apache.spark.scheduler.{SparkListenerBus, SparkListenerEvent}
 import org.apache.spark.status.api.v1
+import org.apache.spark.ui.scope._
 import org.apache.spark.util.{Distribution, Utils}
 
 /**
@@ -33,14 +34,24 @@ import org.apache.spark.util.{Distribution, Utils}
  */
 private[spark] class AppStateStore private (store: KVStore, tempStorePath: Option[File]) {
 
+  def appInfo(): v1.ApplicationInfo = {
+    // There should be a single application info for a UIStore instance, so do no checks here.
+    val it = store.view(classOf[ApplicationInfoWrapper]).closeableIterator()
+    try {
+      it.next().info
+    } finally {
+      it.close()
+    }
+  }
+
   def environmentInfo(): v1.ApplicationEnvironmentInfo = {
     val klass = classOf[ApplicationEnvironmentInfoWrapper]
     store.read(klass, klass.getName()).info
   }
 
   def jobsList(statuses: JList[JobExecutionStatus]): Seq[v1.JobData] = {
-    val it = store.view(classOf[JobDataWrapper]).asScala.map(_.info)
-    if (!statuses.isEmpty()) {
+    val it = store.view(classOf[JobDataWrapper]).reverse().asScala.map(_.info)
+    if (statuses != null && !statuses.isEmpty()) {
       it.filter { job => statuses.contains(job.status) }.toSeq
     } else {
       it.toSeq
@@ -77,8 +88,8 @@ private[spark] class AppStateStore private (store: KVStore, tempStorePath: Optio
   }
 
   def stageList(statuses: JList[v1.StageStatus]): Seq[v1.StageData] = {
-    val it = store.view(classOf[StageDataWrapper]).asScala.map(_.info)
-    if (!statuses.isEmpty) {
+    val it = store.view(classOf[StageDataWrapper]).reverse().asScala.map(_.info)
+    if (statuses != null && !statuses.isEmpty()) {
       it.filter { s => statuses.contains(s.status) }.toSeq
     } else {
       it.toSeq
@@ -89,14 +100,26 @@ private[spark] class AppStateStore private (store: KVStore, tempStorePath: Optio
     val it = store.view(classOf[StageDataWrapper]).index("stageId").first(stageId)
       .closeableIterator()
     try {
-      it.asScala.map(_.info).takeWhile(_.stageId == stageId).toList
+      it.asScala.map(_.info).takeWhile(_.stageId == stageId).map(stageWithDetails).toList
+    } finally {
+      it.close()
+    }
+  }
+
+  def lastStageAttempt(stageId: Int): v1.StageData = {
+    val it = store.view(classOf[StageDataWrapper]).index("stageId").reverse().first(stageId)
+      .closeableIterator()
+    try {
+      it.next().info
     } finally {
       it.close()
     }
   }
 
   def stageAttempt(stageId: Int, stageAttemptId: Int): v1.StageData = {
-    store.read(classOf[StageDataWrapper], Array(stageId, stageAttemptId)).info
+    val stageKey = Array(stageId, stageAttemptId)
+    val stage = store.read(classOf[StageDataWrapper], stageKey).info
+    stageWithDetails(stage)
   }
 
   def taskSummary(
@@ -201,6 +224,12 @@ private[spark] class AppStateStore private (store: KVStore, tempStorePath: Optio
     )
   }
 
+  def taskList(stageId: Int, stageAttemptId: Int, maxTasks: Int): Seq[v1.TaskData] = {
+    val stageKey = Array(stageId, stageAttemptId)
+    val view = store.view(classOf[TaskDataWrapper]).index("stage").first(stageKey).reverse()
+    toTaskList(view, stageKey, 0, maxTasks).reverse
+  }
+
   def taskList(
       stageId: Int,
       stageAttemptId: Int,
@@ -215,17 +244,84 @@ private[spark] class AppStateStore private (store: KVStore, tempStorePath: Optio
       case v1.TaskSorting.INCREASING_RUNTIME =>
         base.index("runtime").first(stageKey ++ Array(-1L))
       case v1.TaskSorting.DECREASING_RUNTIME =>
-        base.index("runtime").first(stageKey ++ Array(-1L)).reverse()
+        base.index("runtime").first(stageKey ++ Array(Long.MaxValue)).reverse()
     }
-    val it = indexed.closeableIterator()
+    toTaskList(indexed, stageKey, offset, length)
+  }
 
+  private def toTaskList(
+      view: KVStoreView[TaskDataWrapper],
+      stageKey: Array[Int],
+      offset: Int,
+      length: Int): Seq[v1.TaskData] = {
+    val it = view.skip(offset).closeableIterator()
     try {
-      it.asScala.takeWhile { t => Arrays.equals(t.stage, stageKey) }.map(_.info).toList
+      var taken = 0
+      it.asScala.takeWhile { t =>
+        val take = taken < length && Arrays.equals(t.stage, stageKey)
+        taken += 1
+        take
+      }.map(_.info).toList
     } finally {
       it.close()
     }
   }
 
+  private def stageWithDetails(stage: v1.StageData): v1.StageData = {
+    // TODO: limit tasks returned.
+    val maxTasks = Int.MaxValue
+    val tasks = taskList(stage.stageId, stage.attemptId, maxTasks)
+      .map { t => (t.taskId, t) }
+      .toMap
+
+    val stageKey = Array(stage.stageId, stage.attemptId)
+    val it = store.view(classOf[ExecutorStageSummaryWrapper]).index("stage").first(stageKey)
+      .closeableIterator()
+    val execs = try {
+      it.asScala
+        .takeWhile { exec => Arrays.equals(exec.stage, stageKey) }
+        .map { exec => (exec.executorId -> exec.info) }
+        .toMap
+    } finally {
+      it.close()
+    }
+
+    new v1.StageData(
+      stage.status,
+      stage.stageId,
+      stage.attemptId,
+      stage.numTasks,
+      stage.numActiveTasks,
+      stage.numCompleteTasks,
+      stage.numFailedTasks,
+      stage.numKilledTasks,
+      stage.executorRunTime,
+      stage.executorCpuTime,
+      stage.submissionTime,
+      stage.firstTaskLaunchedTime,
+      stage.completionTime,
+      stage.failureReason,
+      stage.inputBytes,
+      stage.inputRecords,
+      stage.outputBytes,
+      stage.outputRecords,
+      stage.shuffleReadBytes,
+      stage.shuffleReadRecords,
+      stage.shuffleWriteBytes,
+      stage.shuffleWriteRecords,
+      stage.memoryBytesSpilled,
+      stage.diskBytesSpilled,
+      stage.name,
+      stage.description,
+      stage.details,
+      stage.schedulingPool,
+      stage.rddIds,
+      stage.accumulatorUpdates,
+      Some(tasks),
+      Some(execs),
+      stage.killedTasksSummary)
+  }
+
   def rddList(cachedOnly: Boolean = true): Seq[v1.RDDStorageInfo] = {
     store.view(classOf[RDDStorageInfoWrapper]).asScala.map(_.info).filter { rdd =>
       !cachedOnly || rdd.numCachedPartitions > 0
@@ -244,6 +340,28 @@ private[spark] class AppStateStore private (store: KVStore, tempStorePath: Optio
     store.view(classOf[StreamBlockData]).asScala.toSeq
   }
 
+  def operationGraphForStage(stageId: Int): RDDOperationGraph = {
+    store.read(classOf[RDDOperationGraphWrapper], stageId).toRDDOperationGraph()
+  }
+
+  def operationGraphForJob(jobId: Int): Seq[RDDOperationGraph] = {
+    val job = store.read(classOf[JobDataWrapper], jobId)
+    val stages = job.info.stageIds
+    val skipped = stages.toSet - job.completedStages
+
+    stages.map { id =>
+      val g = store.read(classOf[RDDOperationGraphWrapper], id).toRDDOperationGraph()
+      if (skipped.contains(id) && !g.rootCluster.name.contains("skipped")) {
+        g.rootCluster.setName(g.rootCluster.name + " (skipped)")
+      }
+      g
+    }
+  }
+
+  def pool(name: String): PoolData = {
+    store.read(classOf[PoolData], name)
+  }
+
   def close(): Unit = {
     store.close()
     tempStorePath.foreach(Utils.deleteRecursively)
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllJobsResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllJobsResource.scala
index d0d9ef1165e81..b4fa3e633f6c1 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/AllJobsResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllJobsResource.scala
@@ -22,7 +22,6 @@ import javax.ws.rs.core.MediaType
 
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.ui.SparkUI
-import org.apache.spark.ui.jobs.JobProgressListener
 import org.apache.spark.ui.jobs.UIData.JobUIData
 
 @Produces(Array(MediaType.APPLICATION_JSON))
@@ -30,74 +29,7 @@ private[v1] class AllJobsResource(ui: SparkUI) {
 
   @GET
   def jobsList(@QueryParam("status") statuses: JList[JobExecutionStatus]): Seq[JobData] = {
-    val statusToJobs: Seq[(JobExecutionStatus, Seq[JobUIData])] =
-      AllJobsResource.getStatusToJobs(ui)
-    val adjStatuses: JList[JobExecutionStatus] = {
-      if (statuses.isEmpty) {
-        Arrays.asList(JobExecutionStatus.values(): _*)
-      } else {
-        statuses
-      }
-    }
-    val jobInfos = for {
-      (status, jobs) <- statusToJobs
-      job <- jobs if adjStatuses.contains(status)
-    } yield {
-      AllJobsResource.convertJobData(job, ui.jobProgressListener, false)
-    }
-    jobInfos.sortBy{- _.jobId}
+    ui.store.jobsList(statuses)
   }
 
 }
-
-private[v1] object AllJobsResource {
-
-  def getStatusToJobs(ui: SparkUI): Seq[(JobExecutionStatus, Seq[JobUIData])] = {
-    val statusToJobs = ui.jobProgressListener.synchronized {
-      Seq(
-        JobExecutionStatus.RUNNING -> ui.jobProgressListener.activeJobs.values.toSeq,
-        JobExecutionStatus.SUCCEEDED -> ui.jobProgressListener.completedJobs.toSeq,
-        JobExecutionStatus.FAILED -> ui.jobProgressListener.failedJobs.reverse.toSeq
-      )
-    }
-    statusToJobs
-  }
-
-  def convertJobData(
-      job: JobUIData,
-      listener: JobProgressListener,
-      includeStageDetails: Boolean): JobData = {
-    listener.synchronized {
-      val lastStageInfo =
-        if (job.stageIds.isEmpty) {
-          None
-        } else {
-          listener.stageIdToInfo.get(job.stageIds.max)
-        }
-      val lastStageData = lastStageInfo.flatMap { s =>
-        listener.stageIdToData.get((s.stageId, s.attemptId))
-      }
-      val lastStageName = lastStageInfo.map { _.name }.getOrElse("(Unknown Stage Name)")
-      val lastStageDescription = lastStageData.flatMap { _.description }
-      new JobData(
-        jobId = job.jobId,
-        name = lastStageName,
-        description = lastStageDescription,
-        submissionTime = job.submissionTime.map{new Date(_)},
-        completionTime = job.completionTime.map{new Date(_)},
-        stageIds = job.stageIds,
-        jobGroup = job.jobGroup,
-        status = job.status,
-        numTasks = job.numTasks,
-        numActiveTasks = job.numActiveTasks,
-        numCompletedTasks = job.numCompletedTasks,
-        numSkippedTasks = job.numSkippedTasks,
-        numFailedTasks = job.numFailedTasks,
-        numActiveStages = job.numActiveStages,
-        numCompletedStages = job.completedStageIndices.size,
-        numSkippedStages = job.numSkippedStages,
-        numFailedStages = job.numFailedStages
-      )
-    }
-  }
-}
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
index b81cef207d20e..036a7ba12e4dc 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
@@ -31,285 +31,7 @@ private[v1] class AllStagesResource(ui: SparkUI) {
 
   @GET
   def stageList(@QueryParam("status") statuses: JList[StageStatus]): Seq[StageData] = {
-    val listener = ui.jobProgressListener
-    val stageAndStatus = AllStagesResource.stagesAndStatus(ui)
-    val adjStatuses = {
-      if (statuses.isEmpty()) {
-        Arrays.asList(StageStatus.values(): _*)
-      } else {
-        statuses
-      }
-    }
-    for {
-      (status, stageList) <- stageAndStatus
-      stageInfo: StageInfo <- stageList if adjStatuses.contains(status)
-      stageUiData: StageUIData <- listener.synchronized {
-        listener.stageIdToData.get((stageInfo.stageId, stageInfo.attemptId))
-      }
-    } yield {
-      AllStagesResource.stageUiToStageData(status, stageInfo, stageUiData, includeDetails = false)
-    }
+    ui.store.stageList(statuses)
   }
-}
-
-private[v1] object AllStagesResource {
-  def stageUiToStageData(
-      status: StageStatus,
-      stageInfo: StageInfo,
-      stageUiData: StageUIData,
-      includeDetails: Boolean): StageData = {
-
-    val taskLaunchTimes = stageUiData.taskData.values.map(_.taskInfo.launchTime).filter(_ > 0)
-
-    val firstTaskLaunchedTime: Option[Date] =
-      if (taskLaunchTimes.nonEmpty) {
-        Some(new Date(taskLaunchTimes.min))
-      } else {
-        None
-      }
-
-    val taskData = if (includeDetails) {
-      Some(stageUiData.taskData.map { case (k, v) => k -> convertTaskData(v) }.toMap)
-    } else {
-      None
-    }
-    val executorSummary = if (includeDetails) {
-      Some(stageUiData.executorSummary.map { case (k, summary) =>
-        k -> new ExecutorStageSummary(
-          taskTime = summary.taskTime,
-          failedTasks = summary.failedTasks,
-          succeededTasks = summary.succeededTasks,
-          inputBytes = summary.inputBytes,
-          outputBytes = summary.outputBytes,
-          shuffleRead = summary.shuffleRead,
-          shuffleWrite = summary.shuffleWrite,
-          memoryBytesSpilled = summary.memoryBytesSpilled,
-          diskBytesSpilled = summary.diskBytesSpilled
-        )
-      }.toMap)
-    } else {
-      None
-    }
-
-    val accumulableInfo = stageUiData.accumulables.values.map { convertAccumulableInfo }.toSeq
-
-    new StageData(
-      status = status,
-      stageId = stageInfo.stageId,
-      attemptId = stageInfo.attemptId,
-      numActiveTasks = stageUiData.numActiveTasks,
-      numCompleteTasks = stageUiData.numCompleteTasks,
-      numFailedTasks = stageUiData.numFailedTasks,
-      executorRunTime = stageUiData.executorRunTime,
-      executorCpuTime = stageUiData.executorCpuTime,
-      submissionTime = stageInfo.submissionTime.map(new Date(_)),
-      firstTaskLaunchedTime,
-      completionTime = stageInfo.completionTime.map(new Date(_)),
-      inputBytes = stageUiData.inputBytes,
-      inputRecords = stageUiData.inputRecords,
-      outputBytes = stageUiData.outputBytes,
-      outputRecords = stageUiData.outputRecords,
-      shuffleReadBytes = stageUiData.shuffleReadTotalBytes,
-      shuffleReadRecords = stageUiData.shuffleReadRecords,
-      shuffleWriteBytes = stageUiData.shuffleWriteBytes,
-      shuffleWriteRecords = stageUiData.shuffleWriteRecords,
-      memoryBytesSpilled = stageUiData.memoryBytesSpilled,
-      diskBytesSpilled = stageUiData.diskBytesSpilled,
-      schedulingPool = stageUiData.schedulingPool,
-      name = stageInfo.name,
-      details = stageInfo.details,
-      accumulatorUpdates = accumulableInfo,
-      tasks = taskData,
-      executorSummary = executorSummary
-    )
-  }
-
-  def stagesAndStatus(ui: SparkUI): Seq[(StageStatus, Seq[StageInfo])] = {
-    val listener = ui.jobProgressListener
-    listener.synchronized {
-      Seq(
-        StageStatus.ACTIVE -> listener.activeStages.values.toSeq,
-        StageStatus.COMPLETE -> listener.completedStages.reverse.toSeq,
-        StageStatus.FAILED -> listener.failedStages.reverse.toSeq,
-        StageStatus.PENDING -> listener.pendingStages.values.toSeq
-      )
-    }
-  }
-
-  def convertTaskData(uiData: TaskUIData): TaskData = {
-    new TaskData(
-      taskId = uiData.taskInfo.taskId,
-      index = uiData.taskInfo.index,
-      attempt = uiData.taskInfo.attemptNumber,
-      launchTime = new Date(uiData.taskInfo.launchTime),
-      duration = uiData.taskDuration,
-      executorId = uiData.taskInfo.executorId,
-      host = uiData.taskInfo.host,
-      status = uiData.taskInfo.status,
-      taskLocality = uiData.taskInfo.taskLocality.toString(),
-      speculative = uiData.taskInfo.speculative,
-      accumulatorUpdates = uiData.taskInfo.accumulables.map { convertAccumulableInfo },
-      errorMessage = uiData.errorMessage,
-      taskMetrics = uiData.metrics.map { convertUiTaskMetrics }
-    )
-  }
-
-  def taskMetricDistributions(
-      allTaskData: Iterable[TaskUIData],
-      quantiles: Array[Double]): TaskMetricDistributions = {
-
-    val rawMetrics = allTaskData.flatMap{_.metrics}.toSeq
-
-    def metricQuantiles(f: InternalTaskMetrics => Double): IndexedSeq[Double] =
-      Distribution(rawMetrics.map { d => f(d) }).get.getQuantiles(quantiles)
-
-    // We need to do a lot of similar munging to nested metrics here.  For each one,
-    // we want (a) extract the values for nested metrics (b) make a distribution for each metric
-    // (c) shove the distribution into the right field in our return type and (d) only return
-    // a result if the option is defined for any of the tasks.  MetricHelper is a little util
-    // to make it a little easier to deal w/ all of the nested options.  Mostly it lets us just
-    // implement one "build" method, which just builds the quantiles for each field.
-
-    val inputMetrics: InputMetricDistributions =
-      new MetricHelper[InternalInputMetrics, InputMetricDistributions](rawMetrics, quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): InternalInputMetrics = raw.inputMetrics
-
-        def build: InputMetricDistributions = new InputMetricDistributions(
-          bytesRead = submetricQuantiles(_.bytesRead),
-          recordsRead = submetricQuantiles(_.recordsRead)
-        )
-      }.build
-
-    val outputMetrics: OutputMetricDistributions =
-      new MetricHelper[InternalOutputMetrics, OutputMetricDistributions](rawMetrics, quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): InternalOutputMetrics = raw.outputMetrics
-
-        def build: OutputMetricDistributions = new OutputMetricDistributions(
-          bytesWritten = submetricQuantiles(_.bytesWritten),
-          recordsWritten = submetricQuantiles(_.recordsWritten)
-        )
-      }.build
-
-    val shuffleReadMetrics: ShuffleReadMetricDistributions =
-      new MetricHelper[InternalShuffleReadMetrics, ShuffleReadMetricDistributions](rawMetrics,
-        quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): InternalShuffleReadMetrics =
-          raw.shuffleReadMetrics
-
-        def build: ShuffleReadMetricDistributions = new ShuffleReadMetricDistributions(
-          readBytes = submetricQuantiles(_.totalBytesRead),
-          readRecords = submetricQuantiles(_.recordsRead),
-          remoteBytesRead = submetricQuantiles(_.remoteBytesRead),
-          remoteBlocksFetched = submetricQuantiles(_.remoteBlocksFetched),
-          localBlocksFetched = submetricQuantiles(_.localBlocksFetched),
-          totalBlocksFetched = submetricQuantiles(_.totalBlocksFetched),
-          fetchWaitTime = submetricQuantiles(_.fetchWaitTime)
-        )
-      }.build
-
-    val shuffleWriteMetrics: ShuffleWriteMetricDistributions =
-      new MetricHelper[InternalShuffleWriteMetrics, ShuffleWriteMetricDistributions](rawMetrics,
-        quantiles) {
-        def getSubmetrics(raw: InternalTaskMetrics): InternalShuffleWriteMetrics =
-          raw.shuffleWriteMetrics
 
-        def build: ShuffleWriteMetricDistributions = new ShuffleWriteMetricDistributions(
-          writeBytes = submetricQuantiles(_.bytesWritten),
-          writeRecords = submetricQuantiles(_.recordsWritten),
-          writeTime = submetricQuantiles(_.writeTime)
-        )
-      }.build
-
-    new TaskMetricDistributions(
-      quantiles = quantiles,
-      executorDeserializeTime = metricQuantiles(_.executorDeserializeTime),
-      executorDeserializeCpuTime = metricQuantiles(_.executorDeserializeCpuTime),
-      executorRunTime = metricQuantiles(_.executorRunTime),
-      executorCpuTime = metricQuantiles(_.executorCpuTime),
-      resultSize = metricQuantiles(_.resultSize),
-      jvmGcTime = metricQuantiles(_.jvmGCTime),
-      resultSerializationTime = metricQuantiles(_.resultSerializationTime),
-      memoryBytesSpilled = metricQuantiles(_.memoryBytesSpilled),
-      diskBytesSpilled = metricQuantiles(_.diskBytesSpilled),
-      inputMetrics = inputMetrics,
-      outputMetrics = outputMetrics,
-      shuffleReadMetrics = shuffleReadMetrics,
-      shuffleWriteMetrics = shuffleWriteMetrics
-    )
-  }
-
-  def convertAccumulableInfo(acc: InternalAccumulableInfo): AccumulableInfo = {
-    new AccumulableInfo(
-      acc.id, acc.name.orNull, acc.update.map(_.toString), acc.value.map(_.toString).orNull)
-  }
-
-  def convertUiTaskMetrics(internal: InternalTaskMetrics): TaskMetrics = {
-    new TaskMetrics(
-      executorDeserializeTime = internal.executorDeserializeTime,
-      executorDeserializeCpuTime = internal.executorDeserializeCpuTime,
-      executorRunTime = internal.executorRunTime,
-      executorCpuTime = internal.executorCpuTime,
-      resultSize = internal.resultSize,
-      jvmGcTime = internal.jvmGCTime,
-      resultSerializationTime = internal.resultSerializationTime,
-      memoryBytesSpilled = internal.memoryBytesSpilled,
-      diskBytesSpilled = internal.diskBytesSpilled,
-      inputMetrics = convertInputMetrics(internal.inputMetrics),
-      outputMetrics = convertOutputMetrics(internal.outputMetrics),
-      shuffleReadMetrics = convertShuffleReadMetrics(internal.shuffleReadMetrics),
-      shuffleWriteMetrics = convertShuffleWriteMetrics(internal.shuffleWriteMetrics)
-    )
-  }
-
-  def convertInputMetrics(internal: InternalInputMetrics): InputMetrics = {
-    new InputMetrics(
-      bytesRead = internal.bytesRead,
-      recordsRead = internal.recordsRead
-    )
-  }
-
-  def convertOutputMetrics(internal: InternalOutputMetrics): OutputMetrics = {
-    new OutputMetrics(
-      bytesWritten = internal.bytesWritten,
-      recordsWritten = internal.recordsWritten
-    )
-  }
-
-  def convertShuffleReadMetrics(internal: InternalShuffleReadMetrics): ShuffleReadMetrics = {
-    new ShuffleReadMetrics(
-      remoteBlocksFetched = internal.remoteBlocksFetched,
-      localBlocksFetched = internal.localBlocksFetched,
-      fetchWaitTime = internal.fetchWaitTime,
-      remoteBytesRead = internal.remoteBytesRead,
-      localBytesRead = internal.localBytesRead,
-      recordsRead = internal.recordsRead
-    )
-  }
-
-  def convertShuffleWriteMetrics(internal: InternalShuffleWriteMetrics): ShuffleWriteMetrics = {
-    new ShuffleWriteMetrics(
-      bytesWritten = internal.bytesWritten,
-      writeTime = internal.writeTime,
-      recordsWritten = internal.recordsWritten
-    )
-  }
-}
-
-/**
- * Helper for getting distributions from nested metric types.
- */
-private[v1] abstract class MetricHelper[I, O](
-    rawMetrics: Seq[InternalTaskMetrics],
-    quantiles: Array[Double]) {
-
-  def getSubmetrics(raw: InternalTaskMetrics): I
-
-  def build: O
-
-  val data: Seq[I] = rawMetrics.map(getSubmetrics)
-
-  /** applies the given function to all input metrics, and returns the quantiles */
-  def submetricQuantiles(f: I => Double): IndexedSeq[Double] = {
-    Distribution(data.map { d => f(d) }).get.getQuantiles(quantiles)
-  }
 }
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneJobResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneJobResource.scala
index 653150385c732..3ee884e084c12 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/OneJobResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneJobResource.scala
@@ -16,25 +16,22 @@
  */
 package org.apache.spark.status.api.v1
 
+import java.util.NoSuchElementException
 import javax.ws.rs.{GET, PathParam, Produces}
 import javax.ws.rs.core.MediaType
 
-import org.apache.spark.JobExecutionStatus
 import org.apache.spark.ui.SparkUI
-import org.apache.spark.ui.jobs.UIData.JobUIData
 
 @Produces(Array(MediaType.APPLICATION_JSON))
 private[v1] class OneJobResource(ui: SparkUI) {
 
   @GET
   def oneJob(@PathParam("jobId") jobId: Int): JobData = {
-    val statusToJobs: Seq[(JobExecutionStatus, Seq[JobUIData])] =
-      AllJobsResource.getStatusToJobs(ui)
-    val jobOpt = statusToJobs.flatMap(_._2).find { jobInfo => jobInfo.jobId == jobId}
-    jobOpt.map { job =>
-      AllJobsResource.convertJobData(job, ui.jobProgressListener, false)
-    }.getOrElse {
-      throw new NotFoundException("unknown job: " + jobId)
+    try {
+      ui.store.job(jobId)
+    } catch {
+      case _: NoSuchElementException =>
+        throw new NotFoundException("unknown job: " + jobId)
     }
   }
 
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala
index 3e6d2942d0fbb..2504f2c5d90e5 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/OneStageResource.scala
@@ -24,7 +24,6 @@ import org.apache.spark.scheduler.StageInfo
 import org.apache.spark.status.api.v1.StageStatus._
 import org.apache.spark.status.api.v1.TaskSorting._
 import org.apache.spark.ui.SparkUI
-import org.apache.spark.ui.jobs.JobProgressListener
 import org.apache.spark.ui.jobs.UIData.StageUIData
 
 @Produces(Array(MediaType.APPLICATION_JSON))
@@ -33,11 +32,11 @@ private[v1] class OneStageResource(ui: SparkUI) {
   @GET
   @Path("")
   def stageData(@PathParam("stageId") stageId: Int): Seq[StageData] = {
-    withStage(stageId) { stageAttempts =>
-      stageAttempts.map { stage =>
-        AllStagesResource.stageUiToStageData(stage.status, stage.info, stage.ui,
-          includeDetails = true)
-      }
+    val ret = ui.store.stageData(stageId)
+    if (ret.nonEmpty) {
+      ret
+    } else {
+      throw new NotFoundException(s"unknown stage: $stageId")
     }
   }
 
@@ -46,9 +45,11 @@ private[v1] class OneStageResource(ui: SparkUI) {
   def oneAttemptData(
       @PathParam("stageId") stageId: Int,
       @PathParam("stageAttemptId") stageAttemptId: Int): StageData = {
-    withStageAttempt(stageId, stageAttemptId) { stage =>
-      AllStagesResource.stageUiToStageData(stage.status, stage.info, stage.ui,
-        includeDetails = true)
+    try {
+      ui.store.stageAttempt(stageId, stageAttemptId)
+    } catch {
+      case _: NoSuchElementException =>
+        throw new NotFoundException(s"unknown attempt $stageAttemptId for stage $stageId.")
     }
   }
 
@@ -59,17 +60,16 @@ private[v1] class OneStageResource(ui: SparkUI) {
       @PathParam("stageAttemptId") stageAttemptId: Int,
       @DefaultValue("0.05,0.25,0.5,0.75,0.95") @QueryParam("quantiles") quantileString: String)
   : TaskMetricDistributions = {
-    withStageAttempt(stageId, stageAttemptId) { stage =>
-      val quantiles = quantileString.split(",").map { s =>
-        try {
-          s.toDouble
-        } catch {
-          case nfe: NumberFormatException =>
-            throw new BadParameterException("quantiles", "double", s)
-        }
+    val quantiles = quantileString.split(",").map { s =>
+      try {
+        s.toDouble
+      } catch {
+        case nfe: NumberFormatException =>
+          throw new BadParameterException("quantiles", "double", s)
       }
-      AllStagesResource.taskMetricDistributions(stage.ui.taskData.values, quantiles)
     }
+
+    ui.store.taskSummary(stageId, stageAttemptId, quantiles)
   }
 
   @GET
@@ -80,71 +80,7 @@ private[v1] class OneStageResource(ui: SparkUI) {
       @DefaultValue("0") @QueryParam("offset") offset: Int,
       @DefaultValue("20") @QueryParam("length") length: Int,
       @DefaultValue("ID") @QueryParam("sortBy") sortBy: TaskSorting): Seq[TaskData] = {
-    withStageAttempt(stageId, stageAttemptId) { stage =>
-      val tasks = stage.ui.taskData.values.map{AllStagesResource.convertTaskData}.toIndexedSeq
-        .sorted(OneStageResource.ordering(sortBy))
-      tasks.slice(offset, offset + length)
-    }
-  }
-
-  private case class StageStatusInfoUi(status: StageStatus, info: StageInfo, ui: StageUIData)
-
-  private def withStage[T](stageId: Int)(f: Seq[StageStatusInfoUi] => T): T = {
-    val stageAttempts = findStageStatusUIData(ui.jobProgressListener, stageId)
-    if (stageAttempts.isEmpty) {
-      throw new NotFoundException("unknown stage: " + stageId)
-    } else {
-      f(stageAttempts)
-    }
+    ui.store.taskList(stageId, stageAttemptId, offset, length, sortBy)
   }
 
-  private def findStageStatusUIData(
-      listener: JobProgressListener,
-      stageId: Int): Seq[StageStatusInfoUi] = {
-    listener.synchronized {
-      def getStatusInfoUi(status: StageStatus, infos: Seq[StageInfo]): Seq[StageStatusInfoUi] = {
-        infos.filter { _.stageId == stageId }.map { info =>
-          val ui = listener.stageIdToData.getOrElse((info.stageId, info.attemptId),
-            // this is an internal error -- we should always have uiData
-            throw new SparkException(
-              s"no stage ui data found for stage: ${info.stageId}:${info.attemptId}")
-          )
-          StageStatusInfoUi(status, info, ui)
-        }
-      }
-      getStatusInfoUi(ACTIVE, listener.activeStages.values.toSeq) ++
-        getStatusInfoUi(COMPLETE, listener.completedStages) ++
-        getStatusInfoUi(FAILED, listener.failedStages) ++
-        getStatusInfoUi(PENDING, listener.pendingStages.values.toSeq)
-    }
-  }
-
-  private def withStageAttempt[T](
-      stageId: Int,
-      stageAttemptId: Int)
-      (f: StageStatusInfoUi => T): T = {
-    withStage(stageId) { attempts =>
-        val oneAttempt = attempts.find { stage => stage.info.attemptId == stageAttemptId }
-        oneAttempt match {
-          case Some(stage) =>
-            f(stage)
-          case None =>
-            val stageAttempts = attempts.map { _.info.attemptId }
-            throw new NotFoundException(s"unknown attempt for stage $stageId.  " +
-              s"Found attempts: ${stageAttempts.mkString("[", ",", "]")}")
-        }
-    }
-  }
-}
-
-object OneStageResource {
-  def ordering(taskSorting: TaskSorting): Ordering[TaskData] = {
-    val extractor: (TaskData => Long) = td =>
-      taskSorting match {
-        case ID => td.taskId
-        case INCREASING_RUNTIME => td.taskMetrics.map{_.executorRunTime}.getOrElse(-1L)
-        case DECREASING_RUNTIME => -td.taskMetrics.map{_.executorRunTime}.getOrElse(-1L)
-      }
-    Ordering.by(extractor)
-  }
 }
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 25077d2669d9f..09d220085e8c3 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -61,10 +61,15 @@ class ExecutorStageSummary private[spark](
     val taskTime : Long,
     val failedTasks : Int,
     val succeededTasks : Int,
+    val killedTasks : Int,
     val inputBytes : Long,
+    val inputRecords : Long,
     val outputBytes : Long,
+    val outputRecords : Long,
     val shuffleRead : Long,
+    val shuffleReadRecords : Long,
     val shuffleWrite : Long,
+    val shuffleWriteRecords : Long,
     val memoryBytesSpilled : Long,
     val diskBytesSpilled : Long)
 
@@ -111,10 +116,12 @@ class JobData private[spark](
     val numCompletedTasks: Int,
     val numSkippedTasks: Int,
     val numFailedTasks: Int,
+    val numKilledTasks: Int,
     val numActiveStages: Int,
     val numCompletedStages: Int,
     val numSkippedStages: Int,
-    val numFailedStages: Int)
+    val numFailedStages: Int,
+    val killedTasksSummary: Map[String, Int])
 
 class RDDStorageInfo private[spark](
     val id: Int,
@@ -152,15 +159,18 @@ class StageData private[spark](
     val status: StageStatus,
     val stageId: Int,
     val attemptId: Int,
+    val numTasks: Int,
     val numActiveTasks: Int,
     val numCompleteTasks: Int,
     val numFailedTasks: Int,
+    val numKilledTasks: Int,
 
     val executorRunTime: Long,
     val executorCpuTime: Long,
     val submissionTime: Option[Date],
     val firstTaskLaunchedTime: Option[Date],
     val completionTime: Option[Date],
+    val failureReason: Option[String],
 
     val inputBytes: Long,
     val inputRecords: Long,
@@ -174,18 +184,22 @@ class StageData private[spark](
     val diskBytesSpilled: Long,
 
     val name: String,
+    val description: Option[String],
     val details: String,
     val schedulingPool: String,
 
+    val rddIds: Seq[Int],
     val accumulatorUpdates: Seq[AccumulableInfo],
     val tasks: Option[Map[Long, TaskData]],
-    val executorSummary: Option[Map[String, ExecutorStageSummary]])
+    val executorSummary: Option[Map[String, ExecutorStageSummary]],
+    val killedTasksSummary: Map[String, Int])
 
 class TaskData private[spark](
     val taskId: Long,
     val index: Int,
     val attempt: Int,
     val launchTime: Date,
+    val resultFetchStart: Option[Date],
     @JsonDeserialize(contentAs = classOf[JLong])
     val duration: Option[Long],
     val executorId: String,
@@ -207,6 +221,7 @@ class TaskMetrics private[spark](
     val resultSerializationTime: Long,
     val memoryBytesSpilled: Long,
     val diskBytesSpilled: Long,
+    val peakExecutionMemory: Long,
     val inputMetrics: InputMetrics,
     val outputMetrics: OutputMetrics,
     val shuffleReadMetrics: ShuffleReadMetrics,
diff --git a/core/src/main/scala/org/apache/spark/status/storeTypes.scala b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
index 9db750a169498..5c04041523b54 100644
--- a/core/src/main/scala/org/apache/spark/status/storeTypes.scala
+++ b/core/src/main/scala/org/apache/spark/status/storeTypes.scala
@@ -25,6 +25,7 @@ import org.apache.spark.kvstore.KVIndex
 import org.apache.spark.scheduler.SparkListenerEvent
 import org.apache.spark.status.api.v1._
 import org.apache.spark.status.KVUtils._
+import org.apache.spark.ui.scope._
 
 private[spark] case class AppStatusStoreMetadata(
     val version: Long)
@@ -60,6 +61,15 @@ private[spark] class ExecutorSummaryWrapper(val info: ExecutorSummary) {
 
 }
 
+/**
+ * Used to keep track of the stages a job was initially expected to execute, so that when the
+ * job finishes proper accounting of the actual number of skipped stages and tasks can be done.
+ */
+private[spark] class JobStageSummary(
+    val stageId: Int,
+    val attemptId: Int,
+    val numTasks: Int)
+
 /**
  * Keep track of the existing stages when the job was submitted, and those that were
  * completed during the job's execution. This allows a more accurate acounting of how
@@ -67,7 +77,8 @@ private[spark] class ExecutorSummaryWrapper(val info: ExecutorSummary) {
  */
 private[spark] class JobDataWrapper(
     val info: JobData,
-    val initialStages: Set[(Int, Int)],
+    val initialStages: Seq[JobStageSummary],
+    val submittedStages: Set[Int],
     val completedStages: Set[Int]) {
 
   @JsonIgnore @KVIndex
@@ -158,3 +169,37 @@ private[spark] class StreamBlockData(
   def key: Array[String] = Array(name, executorId)
 
 }
+
+private[spark] class RDDOperationClusterWrapper(
+    val id: String,
+    val name: String,
+    val childNodes: Seq[RDDOperationNode],
+    val childClusters: Seq[RDDOperationClusterWrapper]) {
+
+  def toRDDOperationCluster(): RDDOperationCluster = {
+    val cluster = new RDDOperationCluster(id, name)
+    childNodes.foreach(cluster.attachChildNode)
+    childClusters.foreach { child =>
+      cluster.attachChildCluster(child.toRDDOperationCluster())
+    }
+    cluster
+  }
+
+}
+
+private[spark] class RDDOperationGraphWrapper(
+    @KVIndexParam val stageId: Int,
+    val edges: Seq[RDDOperationEdge],
+    val outgoingEdges: Seq[RDDOperationEdge],
+    val incomingEdges: Seq[RDDOperationEdge],
+    val rootCluster: RDDOperationClusterWrapper) {
+
+  def toRDDOperationGraph(): RDDOperationGraph = {
+    new RDDOperationGraph(edges, outgoingEdges, incomingEdges, rootCluster.toRDDOperationCluster())
+  }
+
+}
+
+private[spark] class PoolData(
+    @KVIndexParam val name: String,
+    val stageIds: Set[Int])
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index ce2696192400c..2e59b9e1514de 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -17,11 +17,11 @@
 
 package org.apache.spark.ui
 
-import java.util.{Date, ServiceLoader}
+import java.util.{Date, List => JList, ServiceLoader}
 
 import scala.collection.JavaConverters._
 
-import org.apache.spark.{SecurityManager, SparkConf, SparkContext}
+import org.apache.spark.{JobExecutionStatus, SecurityManager, SparkConf, SparkContext}
 import org.apache.spark.internal.Logging
 import org.apache.spark.scheduler._
 import org.apache.spark.status.AppStateStore
@@ -29,8 +29,7 @@ import org.apache.spark.status.api.v1._
 import org.apache.spark.ui.JettyUtils._
 import org.apache.spark.ui.env.EnvironmentTab
 import org.apache.spark.ui.exec.ExecutorsTab
-import org.apache.spark.ui.jobs.{JobProgressListener, JobsTab, StagesTab}
-import org.apache.spark.ui.scope.RDDOperationGraphListener
+import org.apache.spark.ui.jobs.{JobsTab, StagesTab}
 import org.apache.spark.ui.storage.StorageTab
 import org.apache.spark.util.Utils
 
@@ -42,8 +41,6 @@ private[spark] class SparkUI private (
     val sc: Option[SparkContext],
     val conf: SparkConf,
     securityManager: SecurityManager,
-    val jobProgressListener: JobProgressListener,
-    val operationGraphListener: RDDOperationGraphListener,
     var appName: String,
     val basePath: String,
     val startTime: Long)
@@ -59,8 +56,8 @@ private[spark] class SparkUI private (
   private var streamingJobProgressListener: Option[SparkListener] = None
 
   /** Initialize all components of the server. */
-  def initialize() {
-    val jobsTab = new JobsTab(this)
+  def initialize(): Unit = {
+    val jobsTab = new JobsTab(this, store)
     attachTab(jobsTab)
     val stagesTab = new StagesTab(this, store)
     attachTab(stagesTab)
@@ -70,6 +67,7 @@ private[spark] class SparkUI private (
     attachHandler(createStaticHandler(SparkUI.STATIC_RESOURCE_DIR, "/static"))
     attachHandler(createRedirectHandler("/", "/jobs/", basePath = basePath))
     attachHandler(ApiRootResource.getServletHandler(this))
+
     // These should be POST only, but, the YARN AM proxy won't proxy POSTs
     attachHandler(createRedirectHandler(
       "/jobs/job/kill", "/jobs/", jobsTab.handleKillRequest, httpMethods = Set("GET", "POST")))
@@ -77,6 +75,7 @@ private[spark] class SparkUI private (
       "/stages/stage/kill", "/stages/", stagesTab.handleKillRequest,
       httpMethods = Set("GET", "POST")))
   }
+
   initialize()
 
   def getSparkUser: String = {
@@ -160,24 +159,11 @@ private[spark] object SparkUI {
       sc: Option[SparkContext],
       store: AppStateStore,
       conf: SparkConf,
-      listenerBus: SparkListenerBus,
       securityManager: SecurityManager,
       appName: String,
       basePath: String,
       startTime: Long): SparkUI = {
-
-    val jobProgressListener = sc.map(_.jobProgressListener).getOrElse {
-      val listener = new JobProgressListener(conf)
-      listenerBus.addListener(listener)
-      listener
-    }
-
-    val operationGraphListener = new RDDOperationGraphListener(conf)
-
-    listenerBus.addListener(operationGraphListener)
-
-    new SparkUI(store, sc, conf, securityManager, jobProgressListener, operationGraphListener,
-      appName, basePath, startTime)
+    new SparkUI(store, sc, conf, securityManager, appName, basePath, startTime)
   }
 
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
index 5818cc364432d..dae9680a94322 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllJobsPage.scala
@@ -23,18 +23,21 @@ import javax.servlet.http.HttpServletRequest
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.{HashMap, ListBuffer}
+import scala.util.Try
 import scala.xml._
 
 import org.apache.commons.lang3.StringEscapeUtils
 
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.scheduler._
+import org.apache.spark.status.AppStateStore
+import org.apache.spark.status.api.v1
 import org.apache.spark.ui._
 import org.apache.spark.ui.jobs.UIData.{JobUIData, StageUIData}
 import org.apache.spark.util.Utils
 
 /** Page showing list of all ongoing and recently finished jobs */
-private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
+private[ui] class AllJobsPage(parent: JobsTab, store: AppStateStore) extends WebUIPage("") {
   private val JOBS_LEGEND =
     <div class="legend-area"><svg width="150px" height="85px">
       <rect class="succeeded-job-legend"
@@ -58,34 +61,20 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
       <text x="35px" y="42px">Removed</text>
     </svg></div>.toString.filter(_ != '\n')
 
-  private def getLastStageNameAndDescription(job: JobUIData): (String, String) = {
-    val lastStageInfo = Option(job.stageIds)
-      .filter(_.nonEmpty)
-      .flatMap { ids => parent.jobProgresslistener.stageIdToInfo.get(ids.max)}
-    val lastStageData = lastStageInfo.flatMap { s =>
-      parent.jobProgresslistener.stageIdToData.get((s.stageId, s.attemptId))
-    }
-    val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
-    val description = lastStageData.flatMap(_.description).getOrElse("")
-    (name, description)
-  }
-
-  private def makeJobEvent(jobUIDatas: Seq[JobUIData]): Seq[String] = {
-    jobUIDatas.filter { jobUIData =>
-      jobUIData.status != JobExecutionStatus.UNKNOWN && jobUIData.submissionTime.isDefined
-    }.map { jobUIData =>
-      val jobId = jobUIData.jobId
-      val status = jobUIData.status
-      val (jobName, jobDescription) = getLastStageNameAndDescription(jobUIData)
+  private def makeJobEvent(jobs: Seq[v1.JobData]): Seq[String] = {
+    jobs.filter { job =>
+      job.status != JobExecutionStatus.UNKNOWN && job.submissionTime.isDefined
+    }.map { job =>
+      val jobId = job.jobId
+      val status = job.status
       val displayJobDescription =
-        if (jobDescription.isEmpty) {
-          jobName
+        if (job.description.isEmpty) {
+          job.name
         } else {
-          UIUtils.makeDescription(jobDescription, "", plainText = true).text
+          UIUtils.makeDescription(job.description.get, "", plainText = true).text
         }
-      val submissionTime = jobUIData.submissionTime.get
-      val completionTimeOpt = jobUIData.completionTime
-      val completionTime = completionTimeOpt.getOrElse(System.currentTimeMillis())
+      val submissionTime = job.submissionTime.get.getTime()
+      val completionTime = job.completionTime.map(_.getTime()).getOrElse(System.currentTimeMillis())
       val classNameByStatus = status match {
         case JobExecutionStatus.SUCCEEDED => "succeeded"
         case JobExecutionStatus.FAILED => "failed"
@@ -170,7 +159,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
   }
 
   private def makeTimeline(
-      jobs: Seq[JobUIData],
+      jobs: Seq[v1.JobData],
       executors: Seq[SparkListenerEvent],
       startTime: Long): Seq[Node] = {
 
@@ -218,7 +207,7 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
       request: HttpServletRequest,
       tableHeaderId: String,
       jobTag: String,
-      jobs: Seq[JobUIData],
+      jobs: Seq[v1.JobData],
       killEnabled: Boolean): Seq[Node] = {
     val allParameters = request.getParameterMap.asScala.toMap
     val parameterOtherTable = allParameters.filterNot(_._1.startsWith(jobTag))
@@ -257,14 +246,13 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
 
     try {
       new JobPagedTable(
+        store,
         jobs,
         tableHeaderId,
         jobTag,
         UIUtils.prependBaseUri(parent.basePath),
         "jobs", // subPath
         parameterOtherTable,
-        parent.jobProgresslistener.stageIdToInfo,
-        parent.jobProgresslistener.stageIdToData,
         killEnabled,
         currentTime,
         jobIdTitle,
@@ -284,106 +272,120 @@ private[ui] class AllJobsPage(parent: JobsTab) extends WebUIPage("") {
   }
 
   def render(request: HttpServletRequest): Seq[Node] = {
-    val listener = parent.jobProgresslistener
-    listener.synchronized {
-      val startTime = listener.startTime
-      val endTime = listener.endTime
-      val activeJobs = listener.activeJobs.values.toSeq
-      val completedJobs = listener.completedJobs.reverse
-      val failedJobs = listener.failedJobs.reverse
-
-      val activeJobsTable =
-        jobsTable(request, "active", "activeJob", activeJobs, killEnabled = parent.killEnabled)
-      val completedJobsTable =
-        jobsTable(request, "completed", "completedJob", completedJobs, killEnabled = false)
-      val failedJobsTable =
-        jobsTable(request, "failed", "failedJob", failedJobs, killEnabled = false)
-
-      val shouldShowActiveJobs = activeJobs.nonEmpty
-      val shouldShowCompletedJobs = completedJobs.nonEmpty
-      val shouldShowFailedJobs = failedJobs.nonEmpty
-
-      val completedJobNumStr = if (completedJobs.size == listener.numCompletedJobs) {
-        s"${completedJobs.size}"
-      } else {
-        s"${listener.numCompletedJobs}, only showing ${completedJobs.size}"
+    val appInfo = store.appInfo()
+    val startTime = appInfo.attempts.head.startTime.getTime()
+    val endTime = appInfo.attempts.head.endTime.getTime()
+
+    val activeJobs = new ListBuffer[v1.JobData]()
+    val _completedJobs = new ListBuffer[v1.JobData]()
+    val _failedJobs = new ListBuffer[v1.JobData]()
+
+    store.jobsList(null).foreach { job =>
+      job.status match {
+        case JobExecutionStatus.SUCCEEDED =>
+          _completedJobs += job
+        case JobExecutionStatus.FAILED =>
+          _failedJobs += job
+        case _ =>
+          activeJobs += job
       }
+    }
 
-      val summary: NodeSeq =
-        <div>
-          <ul class="unstyled">
-            <li>
-              <strong>User:</strong>
-              {parent.getSparkUser}
-            </li>
-            <li>
-              <strong>Total Uptime:</strong>
-              {
-                if (endTime < 0 && parent.sc.isDefined) {
-                  UIUtils.formatDuration(System.currentTimeMillis() - startTime)
-                } else if (endTime > 0) {
-                  UIUtils.formatDuration(endTime - startTime)
-                }
-              }
-            </li>
-            <li>
-              <strong>Scheduling Mode: </strong>
-              {listener.schedulingMode.map(_.toString).getOrElse("Unknown")}
-            </li>
+    val completedJobs = _completedJobs.toSeq.reverse
+    val failedJobs = _failedJobs.toSeq.reverse
+
+    val activeJobsTable =
+      jobsTable(request, "active", "activeJob", activeJobs, killEnabled = parent.killEnabled)
+    val completedJobsTable =
+      jobsTable(request, "completed", "completedJob", completedJobs, killEnabled = false)
+    val failedJobsTable =
+      jobsTable(request, "failed", "failedJob", failedJobs, killEnabled = false)
+
+    val shouldShowActiveJobs = activeJobs.nonEmpty
+    val shouldShowCompletedJobs = completedJobs.nonEmpty
+    val shouldShowFailedJobs = failedJobs.nonEmpty
+
+    val completedJobNumStr = s"${completedJobs.size}"
+    val schedulingMode = store.environmentInfo().sparkProperties.toMap
+      .get("spark.scheduler.mode")
+      .map { mode => SchedulingMode.withName(mode).toString }
+      .getOrElse("Unknown")
+
+    val summary: NodeSeq =
+      <div>
+        <ul class="unstyled">
+          <li>
+            <strong>User:</strong>
+            {parent.getSparkUser}
+          </li>
+          <li>
+            <strong>Total Uptime:</strong>
             {
-              if (shouldShowActiveJobs) {
-                <li>
-                  <a href="#active"><strong>Active Jobs:</strong></a>
-                  {activeJobs.size}
-                </li>
+              if (endTime < 0 && parent.sc.isDefined) {
+                UIUtils.formatDuration(System.currentTimeMillis() - startTime)
+              } else if (endTime > 0) {
+                UIUtils.formatDuration(endTime - startTime)
               }
             }
-            {
-              if (shouldShowCompletedJobs) {
-                <li id="completed-summary">
-                  <a href="#completed"><strong>Completed Jobs:</strong></a>
-                  {completedJobNumStr}
-                </li>
-              }
+          </li>
+          <li>
+            <strong>Scheduling Mode: </strong>
+            {schedulingMode}
+          </li>
+          {
+            if (shouldShowActiveJobs) {
+              <li>
+                <a href="#active"><strong>Active Jobs:</strong></a>
+                {activeJobs.size}
+              </li>
             }
-            {
-              if (shouldShowFailedJobs) {
-                <li>
-                  <a href="#failed"><strong>Failed Jobs:</strong></a>
-                  {listener.numFailedJobs}
-                </li>
-              }
+          }
+          {
+            if (shouldShowCompletedJobs) {
+              <li id="completed-summary">
+                <a href="#completed"><strong>Completed Jobs:</strong></a>
+                {completedJobNumStr}
+              </li>
             }
-          </ul>
-        </div>
+          }
+          {
+            if (shouldShowFailedJobs) {
+              <li>
+                <a href="#failed"><strong>Failed Jobs:</strong></a>
+                {failedJobs.size}
+              </li>
+            }
+          }
+        </ul>
+      </div>
 
-      var content = summary
-      content ++= makeTimeline(activeJobs ++ completedJobs ++ failedJobs,
-          parent.parent.store.executorEvents(), startTime)
+    var content = summary
+    content ++= makeTimeline(activeJobs ++ completedJobs ++ failedJobs,
+        store.executorEvents(), startTime)
 
-      if (shouldShowActiveJobs) {
-        content ++= <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++
-          activeJobsTable
-      }
-      if (shouldShowCompletedJobs) {
-        content ++= <h4 id="completed">Completed Jobs ({completedJobNumStr})</h4> ++
-          completedJobsTable
-      }
-      if (shouldShowFailedJobs) {
-        content ++= <h4 id ="failed">Failed Jobs ({failedJobs.size})</h4> ++
-          failedJobsTable
-      }
+    if (shouldShowActiveJobs) {
+      content ++= <h4 id="active">Active Jobs ({activeJobs.size})</h4> ++
+        activeJobsTable
+    }
+    if (shouldShowCompletedJobs) {
+      content ++= <h4 id="completed">Completed Jobs ({completedJobNumStr})</h4> ++
+        completedJobsTable
+    }
+    if (shouldShowFailedJobs) {
+      content ++= <h4 id ="failed">Failed Jobs ({failedJobs.size})</h4> ++
+        failedJobsTable
+    }
 
-      val helpText = """A job is triggered by an action, like count() or saveAsTextFile().""" +
-        " Click on a job to see information about the stages of tasks inside it."
+    val helpText = """A job is triggered by an action, like count() or saveAsTextFile().""" +
+      " Click on a job to see information about the stages of tasks inside it."
 
-      UIUtils.headerSparkPage("Spark Jobs", content, parent, helpText = Some(helpText))
-    }
+    UIUtils.headerSparkPage("Spark Jobs", content, parent, helpText = Some(helpText))
   }
+
 }
 
 private[ui] class JobTableRowData(
-    val jobData: JobUIData,
+    val jobData: v1.JobData,
     val lastStageName: String,
     val lastStageDescription: String,
     val duration: Long,
@@ -394,9 +396,8 @@ private[ui] class JobTableRowData(
     val detailUrl: String)
 
 private[ui] class JobDataSource(
-    jobs: Seq[JobUIData],
-    stageIdToInfo: HashMap[Int, StageInfo],
-    stageIdToData: HashMap[(Int, Int), StageUIData],
+    store: AppStateStore,
+    jobs: Seq[v1.JobData],
     basePath: String,
     currentTime: Long,
     pageSize: Int,
@@ -417,40 +418,28 @@ private[ui] class JobDataSource(
     r
   }
 
-  private def getLastStageNameAndDescription(job: JobUIData): (String, String) = {
-    val lastStageInfo = Option(job.stageIds)
-      .filter(_.nonEmpty)
-      .flatMap { ids => stageIdToInfo.get(ids.max)}
-    val lastStageData = lastStageInfo.flatMap { s =>
-      stageIdToData.get((s.stageId, s.attemptId))
-    }
-    val name = lastStageInfo.map(_.name).getOrElse("(Unknown Stage Name)")
-    val description = lastStageData.flatMap(_.description).getOrElse("")
-    (name, description)
-  }
-
-  private def jobRow(jobData: JobUIData): JobTableRowData = {
-    val (lastStageName, lastStageDescription) = getLastStageNameAndDescription(jobData)
+  private def jobRow(jobData: v1.JobData): JobTableRowData = {
     val duration: Option[Long] = {
       jobData.submissionTime.map { start =>
-        val end = jobData.completionTime.getOrElse(System.currentTimeMillis())
-        end - start
+        val end = jobData.completionTime.map(_.getTime()).getOrElse(System.currentTimeMillis())
+        end - start.getTime()
       }
     }
     val formattedDuration = duration.map(d => UIUtils.formatDuration(d)).getOrElse("Unknown")
     val submissionTime = jobData.submissionTime
     val formattedSubmissionTime = submissionTime.map(UIUtils.formatDate).getOrElse("Unknown")
-    val jobDescription = UIUtils.makeDescription(lastStageDescription, basePath, plainText = false)
+    val jobDescription = UIUtils.makeDescription(jobData.description.getOrElse(""),
+      basePath, plainText = false)
 
     val detailUrl = "%s/jobs/job?id=%s".format(basePath, jobData.jobId)
 
-    new JobTableRowData (
+    new JobTableRowData(
       jobData,
-      lastStageName,
-      lastStageDescription,
+      jobData.name,
+      jobData.description.getOrElse(jobData.name),
       duration.getOrElse(-1),
       formattedDuration,
-      submissionTime.getOrElse(-1),
+      submissionTime.map(_.getTime()).getOrElse(-1L),
       formattedSubmissionTime,
       jobDescription,
       detailUrl
@@ -479,14 +468,13 @@ private[ui] class JobDataSource(
 
 }
 private[ui] class JobPagedTable(
-    data: Seq[JobUIData],
+    store: AppStateStore,
+    data: Seq[v1.JobData],
     tableHeaderId: String,
     jobTag: String,
     basePath: String,
     subPath: String,
     parameterOtherTable: Iterable[String],
-    stageIdToInfo: HashMap[Int, StageInfo],
-    stageIdToData: HashMap[(Int, Int), StageUIData],
     killEnabled: Boolean,
     currentTime: Long,
     jobIdTitle: String,
@@ -509,9 +497,8 @@ private[ui] class JobPagedTable(
   override def pageNumberFormField: String = jobTag + ".page"
 
   override val dataSource = new JobDataSource(
+    store,
     data,
-    stageIdToInfo,
-    stageIdToData,
     basePath,
     currentTime,
     pageSize,
@@ -623,14 +610,14 @@ private[ui] class JobPagedTable(
       </td>
       <td>{jobTableRow.formattedDuration}</td>
       <td class="stage-progress-cell">
-        {job.completedStageIndices.size}/{job.stageIds.size - job.numSkippedStages}
+        {job.numCompletedStages}/{job.stageIds.size - job.numSkippedStages}
         {if (job.numFailedStages > 0) s"(${job.numFailedStages} failed)"}
         {if (job.numSkippedStages > 0) s"(${job.numSkippedStages} skipped)"}
       </td>
       <td class="progress-cell">
         {UIUtils.makeProgressBar(started = job.numActiveTasks, completed = job.numCompletedTasks,
         failed = job.numFailedTasks, skipped = job.numSkippedTasks,
-        reasonToNumKilled = job.reasonToNumKilled, total = job.numTasks - job.numSkippedTasks)}
+        reasonToNumKilled = job.killedTasksSummary, total = job.numTasks - job.numSkippedTasks)}
       </td>
     </tr>
   }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
index 2b0816e35747d..8eeb9a2026aaa 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/AllStagesPage.scala
@@ -22,120 +22,125 @@ import javax.servlet.http.HttpServletRequest
 import scala.xml.{Node, NodeSeq}
 
 import org.apache.spark.scheduler.Schedulable
+import org.apache.spark.status.PoolData
+import org.apache.spark.status.api.v1._
 import org.apache.spark.ui.{UIUtils, WebUIPage}
 
 /** Page showing list of all ongoing and recently finished stages and pools */
 private[ui] class AllStagesPage(parent: StagesTab) extends WebUIPage("") {
   private val sc = parent.sc
-  private val listener = parent.progressListener
   private def isFairScheduler = parent.isFairScheduler
 
   def render(request: HttpServletRequest): Seq[Node] = {
-    listener.synchronized {
-      val activeStages = listener.activeStages.values.toSeq
-      val pendingStages = listener.pendingStages.values.toSeq
-      val completedStages = listener.completedStages.reverse
-      val numCompletedStages = listener.numCompletedStages
-      val failedStages = listener.failedStages.reverse
-      val numFailedStages = listener.numFailedStages
-      val subPath = "stages"
+    val allStages = parent.store.stageList(null)
 
-      val activeStagesTable =
-        new StageTableBase(request, activeStages, "active", "activeStage", parent.basePath, subPath,
-          parent.progressListener, parent.isFairScheduler,
-          killEnabled = parent.killEnabled, isFailedStage = false)
-      val pendingStagesTable =
-        new StageTableBase(request, pendingStages, "pending", "pendingStage", parent.basePath,
-          subPath, parent.progressListener, parent.isFairScheduler,
-          killEnabled = false, isFailedStage = false)
-      val completedStagesTable =
-        new StageTableBase(request, completedStages, "completed", "completedStage", parent.basePath,
-          subPath, parent.progressListener, parent.isFairScheduler,
-          killEnabled = false, isFailedStage = false)
-      val failedStagesTable =
-        new StageTableBase(request, failedStages, "failed", "failedStage", parent.basePath, subPath,
-          parent.progressListener, parent.isFairScheduler,
-          killEnabled = false, isFailedStage = true)
+    val activeStages = allStages.filter(_.status == StageStatus.ACTIVE)
+    val pendingStages = allStages.filter(_.status == StageStatus.PENDING)
+    val completedStages = allStages.filter(_.status == StageStatus.COMPLETE)
+    val failedStages = allStages.filter(_.status == StageStatus.FAILED).reverse
 
-      // For now, pool information is only accessible in live UIs
-      val pools = sc.map(_.getAllPools).getOrElse(Seq.empty[Schedulable])
-      val poolTable = new PoolTable(pools, parent)
+    val numCompletedStages = completedStages.size
+    val numFailedStages = failedStages.size
+    val subPath = "stages"
 
-      val shouldShowActiveStages = activeStages.nonEmpty
-      val shouldShowPendingStages = pendingStages.nonEmpty
-      val shouldShowCompletedStages = completedStages.nonEmpty
-      val shouldShowFailedStages = failedStages.nonEmpty
+    val activeStagesTable =
+      new StageTableBase(parent.store, request, activeStages, "active", "activeStage",
+        parent.basePath, subPath, parent.isFairScheduler, parent.killEnabled, false)
+    val pendingStagesTable =
+      new StageTableBase(parent.store, request, pendingStages, "pending", "pendingStage",
+        parent.basePath, subPath, parent.isFairScheduler, false, false)
+    val completedStagesTable =
+      new StageTableBase(parent.store, request, completedStages, "completed", "completedStage",
+        parent.basePath, subPath, parent.isFairScheduler, false, false)
+    val failedStagesTable =
+      new StageTableBase(parent.store, request, failedStages, "failed", "failedStage",
+        parent.basePath, subPath, parent.isFairScheduler, false, true)
 
-      val completedStageNumStr = if (numCompletedStages == completedStages.size) {
-        s"$numCompletedStages"
-      } else {
-        s"$numCompletedStages, only showing ${completedStages.size}"
+    // For now, pool information is only accessible in live UIs
+    val pools = sc.map(_.getAllPools).getOrElse(Seq.empty[Schedulable]).map { pool =>
+      val uiPool = try {
+        parent.store.pool(pool.name)
+      } catch {
+        case _: NoSuchElementException =>
+          new PoolData(pool.name, Set())
       }
+      pool -> uiPool
+    }.toMap
+    val poolTable = new PoolTable(pools, parent)
 
-      val summary: NodeSeq =
-        <div>
-          <ul class="unstyled">
-            {
-              if (shouldShowActiveStages) {
-                <li>
-                  <a href="#active"><strong>Active Stages:</strong></a>
-                  {activeStages.size}
-                </li>
-              }
+    val shouldShowActiveStages = activeStages.nonEmpty
+    val shouldShowPendingStages = pendingStages.nonEmpty
+    val shouldShowCompletedStages = completedStages.nonEmpty
+    val shouldShowFailedStages = failedStages.nonEmpty
+
+    val completedStageNumStr = if (numCompletedStages == completedStages.size) {
+      s"$numCompletedStages"
+    } else {
+      s"$numCompletedStages, only showing ${completedStages.size}"
+    }
+
+    val summary: NodeSeq =
+      <div>
+        <ul class="unstyled">
+          {
+            if (shouldShowActiveStages) {
+              <li>
+                <a href="#active"><strong>Active Stages:</strong></a>
+                {activeStages.size}
+              </li>
             }
-            {
-              if (shouldShowPendingStages) {
-                <li>
-                  <a href="#pending"><strong>Pending Stages:</strong></a>
-                  {pendingStages.size}
-                </li>
-              }
+          }
+          {
+            if (shouldShowPendingStages) {
+              <li>
+                <a href="#pending"><strong>Pending Stages:</strong></a>
+                {pendingStages.size}
+              </li>
             }
-            {
-              if (shouldShowCompletedStages) {
-                <li id="completed-summary">
-                  <a href="#completed"><strong>Completed Stages:</strong></a>
-                  {completedStageNumStr}
-                </li>
-              }
+          }
+          {
+            if (shouldShowCompletedStages) {
+              <li id="completed-summary">
+                <a href="#completed"><strong>Completed Stages:</strong></a>
+                {completedStageNumStr}
+              </li>
             }
-            {
-              if (shouldShowFailedStages) {
-                <li>
-                  <a href="#failed"><strong>Failed Stages:</strong></a>
-                  {numFailedStages}
-                </li>
-              }
+          }
+          {
+            if (shouldShowFailedStages) {
+              <li>
+                <a href="#failed"><strong>Failed Stages:</strong></a>
+                {numFailedStages}
+              </li>
             }
-          </ul>
-        </div>
-
-      var content = summary ++
-        {
-          if (sc.isDefined && isFairScheduler) {
-            <h4>{pools.size} Fair Scheduler Pools</h4> ++ poolTable.toNodeSeq
-          } else {
-            Seq[Node]()
           }
+        </ul>
+      </div>
+
+    var content = summary ++
+      {
+        if (sc.isDefined && isFairScheduler) {
+          <h4>{pools.size} Fair Scheduler Pools</h4> ++ poolTable.toNodeSeq
+        } else {
+          Seq[Node]()
         }
-      if (shouldShowActiveStages) {
-        content ++= <h4 id="active">Active Stages ({activeStages.size})</h4> ++
-        activeStagesTable.toNodeSeq
       }
-      if (shouldShowPendingStages) {
-        content ++= <h4 id="pending">Pending Stages ({pendingStages.size})</h4> ++
-        pendingStagesTable.toNodeSeq
-      }
-      if (shouldShowCompletedStages) {
-        content ++= <h4 id="completed">Completed Stages ({completedStageNumStr})</h4> ++
-        completedStagesTable.toNodeSeq
-      }
-      if (shouldShowFailedStages) {
-        content ++= <h4 id ="failed">Failed Stages ({numFailedStages})</h4> ++
-        failedStagesTable.toNodeSeq
-      }
-      UIUtils.headerSparkPage("Stages for All Jobs", content, parent)
+    if (shouldShowActiveStages) {
+      content ++= <h4 id="active">Active Stages ({activeStages.size})</h4> ++
+      activeStagesTable.toNodeSeq
+    }
+    if (shouldShowPendingStages) {
+      content ++= <h4 id="pending">Pending Stages ({pendingStages.size})</h4> ++
+      pendingStagesTable.toNodeSeq
     }
+    if (shouldShowCompletedStages) {
+      content ++= <h4 id="completed">Completed Stages ({completedStageNumStr})</h4> ++
+      completedStagesTable.toNodeSeq
+    }
+    if (shouldShowFailedStages) {
+      content ++= <h4 id ="failed">Failed Stages ({numFailedStages})</h4> ++
+      failedStagesTable.toNodeSeq
+    }
+    UIUtils.headerSparkPage("Stages for All Jobs", content, parent)
   }
 }
-
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
index 7cd625925c2c4..a82f591d79826 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/ExecutorTable.scala
@@ -17,42 +17,36 @@
 
 package org.apache.spark.ui.jobs
 
+import java.util.NoSuchElementException
+
 import scala.collection.mutable
 import scala.xml.{Node, Unparsed}
 
 import org.apache.spark.status.AppStateStore
+import org.apache.spark.status.api.v1.StageData
 import org.apache.spark.ui.{ToolTips, UIUtils}
-import org.apache.spark.ui.jobs.UIData.StageUIData
 import org.apache.spark.util.Utils
 
 /** Stage summary grouped by executors. */
-private[ui] class ExecutorTable(
-    stageId: Int,
-    stageAttemptId: Int,
-    parent: StagesTab,
-    store: AppStateStore) {
-  private val listener = parent.progressListener
+private[ui] class ExecutorTable(stageId: Int, stageAttemptId: Int, store: AppStateStore) {
 
   def toNodeSeq: Seq[Node] = {
-    listener.synchronized {
-      executorTable()
+    val stageData = try {
+      Some(store.stageAttempt(stageId, stageAttemptId))
+    } catch {
+      case _: NoSuchElementException => None
     }
-  }
-
-  /** Special table which merges two header cells. */
-  private def executorTable[T](): Seq[Node] = {
-    val stageData = listener.stageIdToData.get((stageId, stageAttemptId))
     var hasInput = false
     var hasOutput = false
     var hasShuffleWrite = false
     var hasShuffleRead = false
     var hasBytesSpilled = false
     stageData.foreach { data =>
-        hasInput = data.hasInput
-        hasOutput = data.hasOutput
-        hasShuffleRead = data.hasShuffleRead
-        hasShuffleWrite = data.hasShuffleWrite
-        hasBytesSpilled = data.hasBytesSpilled
+        hasInput = data.inputBytes > 0
+        hasOutput = data.outputBytes > 0
+        hasShuffleRead = data.shuffleReadBytes > 0
+        hasShuffleWrite = data.shuffleWriteBytes > 0
+        hasBytesSpilled = (data.memoryBytesSpilled + data.diskBytesSpilled) > 0
     }
 
     <table class={UIUtils.TABLE_CLASS_STRIPED_SORTABLE}>
@@ -97,7 +91,7 @@ private[ui] class ExecutorTable(
         </th>
       </thead>
       <tbody>
-        {createExecutorTable()}
+        {createExecutorTable(stageData)}
       </tbody>
     </table>
     <script>
@@ -111,56 +105,49 @@ private[ui] class ExecutorTable(
     </script>
   }
 
-  private def createExecutorTable() : Seq[Node] = {
-    // Make an executor-id -> address map
-    val executorIdToAddress = mutable.HashMap[String, String]()
-    listener.blockManagerIds.foreach { blockManagerId =>
-      val address = blockManagerId.hostPort
-      val executorId = blockManagerId.executorId
-      executorIdToAddress.put(executorId, address)
-    }
-
-    listener.stageIdToData.get((stageId, stageAttemptId)) match {
-      case Some(stageData: StageUIData) =>
-        stageData.executorSummary.toSeq.sortBy(_._1).map { case (k, v) =>
+  private def createExecutorTable(stageDataOpt: Option[StageData]) : Seq[Node] = {
+    stageDataOpt match {
+      case Some(stageData) =>
+        stageData.executorSummary.getOrElse(Map()).toSeq.sortBy(_._1).map { case (k, v) =>
+          val executor = store.executorSummary(k)
           <tr>
             <td>
               <div style="float: left">{k}</div>
               <div style="float: right">
               {
-                store.executorSummary(k).map(_.executorLogs).getOrElse(Map.empty).map {
+                executor.map(_.executorLogs).getOrElse(Map.empty).map {
                   case (logName, logUrl) => <div><a href={logUrl}>{logName}</a></div>
                 }
               }
               </div>
             </td>
-            <td>{executorIdToAddress.getOrElse(k, "CANNOT FIND ADDRESS")}</td>
+            <td>{executor.map { e => e.hostPort }.getOrElse(k, "CANNOT FIND ADDRESS")}</td>
             <td sorttable_customkey={v.taskTime.toString}>{UIUtils.formatDuration(v.taskTime)}</td>
-            <td>{v.failedTasks + v.succeededTasks + v.reasonToNumKilled.values.sum}</td>
+            <td>{v.failedTasks + v.succeededTasks + v.killedTasks}</td>
             <td>{v.failedTasks}</td>
-            <td>{v.reasonToNumKilled.values.sum}</td>
+            <td>{v.killedTasks}</td>
             <td>{v.succeededTasks}</td>
-            {if (stageData.hasInput) {
+            {if (stageData.inputBytes > 0) {
               <td sorttable_customkey={v.inputBytes.toString}>
                 {s"${Utils.bytesToString(v.inputBytes)} / ${v.inputRecords}"}
               </td>
             }}
-            {if (stageData.hasOutput) {
+            {if (stageData.outputBytes > 0) {
               <td sorttable_customkey={v.outputBytes.toString}>
                 {s"${Utils.bytesToString(v.outputBytes)} / ${v.outputRecords}"}
               </td>
             }}
-            {if (stageData.hasShuffleRead) {
+            {if (stageData.shuffleReadBytes > 0) {
               <td sorttable_customkey={v.shuffleRead.toString}>
                 {s"${Utils.bytesToString(v.shuffleRead)} / ${v.shuffleReadRecords}"}
               </td>
             }}
-            {if (stageData.hasShuffleWrite) {
+            {if (stageData.shuffleWriteBytes > 0) {
               <td sorttable_customkey={v.shuffleWrite.toString}>
                 {s"${Utils.bytesToString(v.shuffleWrite)} / ${v.shuffleWriteRecords}"}
               </td>
             }}
-            {if (stageData.hasBytesSpilled) {
+            {if ((stageData.memoryBytesSpilled + stageData.diskBytesSpilled) > 0) {
               <td sorttable_customkey={v.memoryBytesSpilled.toString}>
                 {Utils.bytesToString(v.memoryBytesSpilled)}
               </td>
@@ -168,7 +155,7 @@ private[ui] class ExecutorTable(
                 {Utils.bytesToString(v.diskBytesSpilled)}
               </td>
             }}
-            <td>{v.isBlacklisted}</td>
+            <td>{executor.map(_.isBlacklisted).getOrElse(false)}</td>
           </tr>
         }
       case None =>
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
index dd7e11fef1ba6..976fd4e611d06 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobPage.scala
@@ -21,16 +21,19 @@ import java.util.{Date, Locale}
 import javax.servlet.http.HttpServletRequest
 
 import scala.collection.mutable.{Buffer, ListBuffer}
+import scala.util.Try
 import scala.xml.{Node, NodeSeq, Unparsed, Utility}
 
 import org.apache.commons.lang3.StringEscapeUtils
 
 import org.apache.spark.JobExecutionStatus
 import org.apache.spark.scheduler._
-import org.apache.spark.ui.{ToolTips, UIUtils, WebUIPage}
+import org.apache.spark.status.AppStateStore
+import org.apache.spark.status.api.v1
+import org.apache.spark.ui._
 
 /** Page showing statistics and stage list for a given job */
-private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
+private[ui] class JobPage(parent: JobsTab, store: AppStateStore) extends WebUIPage("job") {
 
   private val STAGES_LEGEND =
     <div class="legend-area"><svg width="150px" height="85px">
@@ -55,14 +58,15 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
       <text x="35px" y="42px">Removed</text>
     </svg></div>.toString.filter(_ != '\n')
 
-  private def makeStageEvent(stageInfos: Seq[StageInfo]): Seq[String] = {
+  private def makeStageEvent(stageInfos: Seq[v1.StageData]): Seq[String] = {
     stageInfos.map { stage =>
       val stageId = stage.stageId
       val attemptId = stage.attemptId
       val name = stage.name
-      val status = stage.getStatusString
-      val submissionTime = stage.submissionTime.get
-      val completionTime = stage.completionTime.getOrElse(System.currentTimeMillis())
+      val status = stage.status.toString
+      val submissionTime = stage.submissionTime.get.getTime()
+      val completionTime = stage.completionTime.map(_.getTime())
+        .getOrElse(System.currentTimeMillis())
 
       // The timeline library treats contents as HTML, so we have to escape them. We need to add
       // extra layers of escaping in order to embed this in a Javascript string literal.
@@ -78,10 +82,10 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
          |   'data-placement="top" data-html="true"' +
          |   'data-title="${jsEscapedName} (Stage ${stageId}.${attemptId})<br>' +
          |   'Status: ${status.toUpperCase(Locale.ROOT)}<br>' +
-         |   'Submitted: ${UIUtils.formatDate(new Date(submissionTime))}' +
+         |   'Submitted: ${UIUtils.formatDate(submissionTime)}' +
          |   '${
                  if (status != "running") {
-                   s"""<br>Completed: ${UIUtils.formatDate(new Date(completionTime))}"""
+                   s"""<br>Completed: ${UIUtils.formatDate(completionTime)}"""
                  } else {
                    ""
                  }
@@ -139,7 +143,7 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
   }
 
   private def makeTimeline(
-      stages: Seq[StageInfo],
+      stages: Seq[v1.StageData],
       executors: Seq[SparkListenerEvent],
       appStartTime: Long): Seq[Node] = {
 
@@ -184,172 +188,180 @@ private[ui] class JobPage(parent: JobsTab) extends WebUIPage("job") {
   }
 
   def render(request: HttpServletRequest): Seq[Node] = {
-    val listener = parent.jobProgresslistener
+    val parameterId = request.getParameter("id")
+    require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
 
-    listener.synchronized {
-      val parameterId = request.getParameter("id")
-      require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
-
-      val jobId = parameterId.toInt
-      val jobDataOption = listener.jobIdToData.get(jobId)
-      if (jobDataOption.isEmpty) {
-        val content =
-          <div id="no-info">
-            <p>No information to display for job {jobId}</p>
-          </div>
-        return UIUtils.headerSparkPage(
-          s"Details for Job $jobId", content, parent)
-      }
-      val jobData = jobDataOption.get
-      val isComplete = jobData.status != JobExecutionStatus.RUNNING
-      val stages = jobData.stageIds.map { stageId =>
-        // This could be empty if the JobProgressListener hasn't received information about the
-        // stage or if the stage information has been garbage collected
-        listener.stageIdToInfo.getOrElse(stageId,
-          new StageInfo(stageId, 0, "Unknown", 0, Seq.empty, Seq.empty, "Unknown"))
+    val jobId = parameterId.toInt
+    val jobDataOption = Try(store.job(jobId)).toOption
+    if (jobDataOption.isEmpty) {
+      val content =
+        <div id="no-info">
+          <p>No information to display for job {jobId}</p>
+        </div>
+      return UIUtils.headerSparkPage(
+        s"Details for Job $jobId", content, parent)
+    }
+    val jobData = jobDataOption.get
+    val isComplete = jobData.status != JobExecutionStatus.RUNNING
+    val stages = jobData.stageIds.map { stageId =>
+      // This could be empty if the listener hasn't received information about the
+      // stage or if the stage information has been garbage collected
+      store.stageData(stageId).lastOption.getOrElse {
+        new v1.StageData(
+          v1.StageStatus.PENDING,
+          stageId,
+          0, 0, 0, 0, 0, 0, 0L, 0L, None, None, None, None, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L, 0L,
+          "Unknown",
+          None,
+          "Unknown",
+          null,
+          Nil,
+          Nil,
+          None,
+          None,
+          Map())
       }
+    }
 
-      val activeStages = Buffer[StageInfo]()
-      val completedStages = Buffer[StageInfo]()
-      // If the job is completed, then any pending stages are displayed as "skipped":
-      val pendingOrSkippedStages = Buffer[StageInfo]()
-      val failedStages = Buffer[StageInfo]()
-      for (stage <- stages) {
-        if (stage.submissionTime.isEmpty) {
-          pendingOrSkippedStages += stage
-        } else if (stage.completionTime.isDefined) {
-          if (stage.failureReason.isDefined) {
-            failedStages += stage
-          } else {
-            completedStages += stage
-          }
+    val activeStages = Buffer[v1.StageData]()
+    val completedStages = Buffer[v1.StageData]()
+    // If the job is completed, then any pending stages are displayed as "skipped":
+    val pendingOrSkippedStages = Buffer[v1.StageData]()
+    val failedStages = Buffer[v1.StageData]()
+    for (stage <- stages) {
+      if (stage.submissionTime.isEmpty) {
+        pendingOrSkippedStages += stage
+      } else if (stage.completionTime.isDefined) {
+        if (stage.status == v1.StageStatus.FAILED) {
+          failedStages += stage
         } else {
-          activeStages += stage
+          completedStages += stage
         }
+      } else {
+        activeStages += stage
       }
+    }
 
-      val basePath = "jobs/job"
+    val basePath = "jobs/job"
 
-      val pendingOrSkippedTableId =
-        if (isComplete) {
-          "pending"
-        } else {
-          "skipped"
-        }
+    val pendingOrSkippedTableId =
+      if (isComplete) {
+        "pending"
+      } else {
+        "skipped"
+      }
 
-      val activeStagesTable =
-        new StageTableBase(request, activeStages, "active", "activeStage", parent.basePath,
-          basePath, parent.jobProgresslistener, parent.isFairScheduler,
-          killEnabled = parent.killEnabled, isFailedStage = false)
-      val pendingOrSkippedStagesTable =
-        new StageTableBase(request, pendingOrSkippedStages, pendingOrSkippedTableId, "pendingStage",
-          parent.basePath, basePath, parent.jobProgresslistener, parent.isFairScheduler,
-          killEnabled = false, isFailedStage = false)
-      val completedStagesTable =
-        new StageTableBase(request, completedStages, "completed", "completedStage", parent.basePath,
-          basePath, parent.jobProgresslistener, parent.isFairScheduler,
-          killEnabled = false, isFailedStage = false)
-      val failedStagesTable =
-        new StageTableBase(request, failedStages, "failed", "failedStage", parent.basePath,
-          basePath, parent.jobProgresslistener, parent.isFairScheduler,
-          killEnabled = false, isFailedStage = true)
+    val activeStagesTable =
+      new StageTableBase(store, request, activeStages, "active", "activeStage", parent.basePath,
+        basePath, parent.isFairScheduler,
+        killEnabled = parent.killEnabled, isFailedStage = false)
+    val pendingOrSkippedStagesTable =
+      new StageTableBase(store, request, pendingOrSkippedStages, pendingOrSkippedTableId,
+        "pendingStage", parent.basePath, basePath, parent.isFairScheduler,
+        killEnabled = false, isFailedStage = false)
+    val completedStagesTable =
+      new StageTableBase(store, request, completedStages, "completed", "completedStage",
+        parent.basePath, basePath, parent.isFairScheduler,
+        killEnabled = false, isFailedStage = false)
+    val failedStagesTable =
+      new StageTableBase(store, request, failedStages, "failed", "failedStage", parent.basePath,
+        basePath, parent.isFairScheduler,
+        killEnabled = false, isFailedStage = true)
 
-      val shouldShowActiveStages = activeStages.nonEmpty
-      val shouldShowPendingStages = !isComplete && pendingOrSkippedStages.nonEmpty
-      val shouldShowCompletedStages = completedStages.nonEmpty
-      val shouldShowSkippedStages = isComplete && pendingOrSkippedStages.nonEmpty
-      val shouldShowFailedStages = failedStages.nonEmpty
+    val shouldShowActiveStages = activeStages.nonEmpty
+    val shouldShowPendingStages = !isComplete && pendingOrSkippedStages.nonEmpty
+    val shouldShowCompletedStages = completedStages.nonEmpty
+    val shouldShowSkippedStages = isComplete && pendingOrSkippedStages.nonEmpty
+    val shouldShowFailedStages = failedStages.nonEmpty
 
-      val summary: NodeSeq =
-        <div>
-          <ul class="unstyled">
-            <li>
-              <Strong>Status:</Strong>
-              {jobData.status}
-            </li>
-            {
-              if (jobData.jobGroup.isDefined) {
-                <li>
-                  <strong>Job Group:</strong>
-                  {jobData.jobGroup.get}
-                </li>
-              }
-            }
-            {
-              if (shouldShowActiveStages) {
-                <li>
-                  <a href="#active"><strong>Active Stages:</strong></a>
-                  {activeStages.size}
-                </li>
-              }
-            }
-            {
-              if (shouldShowPendingStages) {
-                <li>
-                  <a href="#pending">
-                    <strong>Pending Stages:</strong>
-                  </a>{pendingOrSkippedStages.size}
-                </li>
-              }
+    val summary: NodeSeq =
+      <div>
+        <ul class="unstyled">
+          <li>
+            <Strong>Status:</Strong>
+            {jobData.status}
+          </li>
+          {
+            if (jobData.jobGroup.isDefined) {
+              <li>
+                <strong>Job Group:</strong>
+                {jobData.jobGroup.get}
+              </li>
             }
-            {
-              if (shouldShowCompletedStages) {
-                <li>
-                  <a href="#completed"><strong>Completed Stages:</strong></a>
-                  {completedStages.size}
-                </li>
-              }
+          }
+          {
+            if (shouldShowActiveStages) {
+              <li>
+                <a href="#active"><strong>Active Stages:</strong></a>
+                {activeStages.size}
+              </li>
             }
-            {
-              if (shouldShowSkippedStages) {
+          }
+          {
+            if (shouldShowPendingStages) {
               <li>
-                <a href="#skipped"><strong>Skipped Stages:</strong></a>
-                {pendingOrSkippedStages.size}
+                <a href="#pending">
+                  <strong>Pending Stages:</strong>
+                </a>{pendingOrSkippedStages.size}
               </li>
             }
+          }
+          {
+            if (shouldShowCompletedStages) {
+              <li>
+                <a href="#completed"><strong>Completed Stages:</strong></a>
+                {completedStages.size}
+              </li>
             }
-            {
-              if (shouldShowFailedStages) {
-                <li>
-                  <a href="#failed"><strong>Failed Stages:</strong></a>
-                  {failedStages.size}
-                </li>
-              }
+          }
+          {
+            if (shouldShowSkippedStages) {
+            <li>
+              <a href="#skipped"><strong>Skipped Stages:</strong></a>
+              {pendingOrSkippedStages.size}
+            </li>
+          }
+          }
+          {
+            if (shouldShowFailedStages) {
+              <li>
+                <a href="#failed"><strong>Failed Stages:</strong></a>
+                {failedStages.size}
+              </li>
             }
-          </ul>
-        </div>
+          }
+        </ul>
+      </div>
 
-      var content = summary
-      val appStartTime = listener.startTime
-      val operationGraphListener = parent.operationGraphListener
+    var content = summary
+    val appStartTime = store.appInfo().attempts.head.startTime.getTime()
 
-      content ++= makeTimeline(activeStages ++ completedStages ++ failedStages,
-          parent.parent.store.executorEvents(), appStartTime)
+    content ++= makeTimeline(activeStages ++ completedStages ++ failedStages,
+        store.executorEvents(), appStartTime)
 
-      content ++= UIUtils.showDagVizForJob(
-        jobId, operationGraphListener.getOperationGraphForJob(jobId))
+    content ++= UIUtils.showDagVizForJob(
+      jobId, store.operationGraphForJob(jobId))
 
-      if (shouldShowActiveStages) {
-        content ++= <h4 id="active">Active Stages ({activeStages.size})</h4> ++
-          activeStagesTable.toNodeSeq
-      }
-      if (shouldShowPendingStages) {
-        content ++= <h4 id="pending">Pending Stages ({pendingOrSkippedStages.size})</h4> ++
-          pendingOrSkippedStagesTable.toNodeSeq
-      }
-      if (shouldShowCompletedStages) {
-        content ++= <h4 id="completed">Completed Stages ({completedStages.size})</h4> ++
-          completedStagesTable.toNodeSeq
-      }
-      if (shouldShowSkippedStages) {
-        content ++= <h4 id="skipped">Skipped Stages ({pendingOrSkippedStages.size})</h4> ++
-          pendingOrSkippedStagesTable.toNodeSeq
-      }
-      if (shouldShowFailedStages) {
-        content ++= <h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++
-          failedStagesTable.toNodeSeq
-      }
-      UIUtils.headerSparkPage(s"Details for Job $jobId", content, parent, showVisualization = true)
+    if (shouldShowActiveStages) {
+      content ++= <h4 id="active">Active Stages ({activeStages.size})</h4> ++
+        activeStagesTable.toNodeSeq
+    }
+    if (shouldShowPendingStages) {
+      content ++= <h4 id="pending">Pending Stages ({pendingOrSkippedStages.size})</h4> ++
+        pendingOrSkippedStagesTable.toNodeSeq
+    }
+    if (shouldShowCompletedStages) {
+      content ++= <h4 id="completed">Completed Stages ({completedStages.size})</h4> ++
+        completedStagesTable.toNodeSeq
+    }
+    if (shouldShowSkippedStages) {
+      content ++= <h4 id="skipped">Skipped Stages ({pendingOrSkippedStages.size})</h4> ++
+        pendingOrSkippedStagesTable.toNodeSeq
+    }
+    if (shouldShowFailedStages) {
+      content ++= <h4 id ="failed">Failed Stages ({failedStages.size})</h4> ++
+        failedStagesTable.toNodeSeq
     }
+    UIUtils.headerSparkPage(s"Details for Job $jobId", content, parent, showVisualization = true)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
index 13b2ba11f6bc6..74ccc1a75417a 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/JobsTab.scala
@@ -19,34 +19,50 @@ package org.apache.spark.ui.jobs
 
 import javax.servlet.http.HttpServletRequest
 
+import scala.collection.JavaConverters._
+
+import org.apache.spark.JobExecutionStatus
 import org.apache.spark.scheduler.SchedulingMode
-import org.apache.spark.ui.{SparkUI, SparkUITab}
+import org.apache.spark.status.AppStateStore
+import org.apache.spark.ui._
 
 /** Web UI showing progress status of all jobs in the given SparkContext. */
-private[ui] class JobsTab(val parent: SparkUI) extends SparkUITab(parent, "jobs") {
+private[ui] class JobsTab(parent: SparkUI, store: AppStateStore)
+  extends SparkUITab(parent, "jobs") {
+
   val sc = parent.sc
   val killEnabled = parent.killEnabled
-  val jobProgresslistener = parent.jobProgressListener
-  val operationGraphListener = parent.operationGraphListener
 
-  def isFairScheduler: Boolean =
-    jobProgresslistener.schedulingMode == Some(SchedulingMode.FAIR)
+  def isFairScheduler: Boolean = {
+    val configName = "spark.scheduler.mode"
+    val config = sc match {
+      case Some(_sc) =>
+        _sc.conf.getOption(configName)
+      case _ =>
+        store.environmentInfo().sparkProperties.toMap.get(configName)
+    }
+    config.map(SchedulingMode.withName) == Some(SchedulingMode.FAIR)
+  }
 
   def getSparkUser: String = parent.getSparkUser
 
-  attachPage(new AllJobsPage(this))
-  attachPage(new JobPage(this))
+  attachPage(new AllJobsPage(this, store))
+  attachPage(new JobPage(this, store))
 
   def handleKillRequest(request: HttpServletRequest): Unit = {
     if (killEnabled && parent.securityManager.checkModifyPermissions(request.getRemoteUser)) {
-      val jobId = Option(request.getParameter("id")).map(_.toInt)
-      jobId.foreach { id =>
-        if (jobProgresslistener.activeJobs.contains(id)) {
-          sc.foreach(_.cancelJob(id))
-          // Do a quick pause here to give Spark time to kill the job so it shows up as
-          // killed after the refresh. Note that this will block the serving thread so the
-          // time should be limited in duration.
-          Thread.sleep(100)
+      Option(request.getParameter("id")).map(_.toInt).foreach { id =>
+        try {
+          val job = store.job(id)
+          if (job.status == JobExecutionStatus.RUNNING) {
+            sc.foreach(_.cancelJob(id))
+            // Do a quick pause here to give Spark time to kill the job so it shows up as
+            // killed after the refresh. Note that this will block the serving thread so the
+            // time should be limited in duration.
+            Thread.sleep(100)
+          }
+        } catch {
+          case _: NoSuchElementException =>
         }
       }
     }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
index 8ee70d27cc09f..fcb56569f3cc9 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolPage.scala
@@ -17,49 +17,50 @@
 
 package org.apache.spark.ui.jobs
 
+import java.util.NoSuchElementException
 import javax.servlet.http.HttpServletRequest
 
 import scala.xml.Node
 
-import org.apache.spark.scheduler.StageInfo
+import org.apache.spark.status.PoolData
+import org.apache.spark.status.api.v1._
 import org.apache.spark.ui.{UIUtils, WebUIPage}
 
 /** Page showing specific pool details */
 private[ui] class PoolPage(parent: StagesTab) extends WebUIPage("pool") {
-  private val sc = parent.sc
-  private val listener = parent.progressListener
 
   def render(request: HttpServletRequest): Seq[Node] = {
-    listener.synchronized {
-      val poolName = Option(request.getParameter("poolname")).map { poolname =>
-        UIUtils.decodeURLParameter(poolname)
-      }.getOrElse {
-        throw new IllegalArgumentException(s"Missing poolname parameter")
-      }
+    val poolName = Option(request.getParameter("poolname")).map { poolname =>
+      UIUtils.decodeURLParameter(poolname)
+    }.getOrElse {
+      throw new IllegalArgumentException(s"Missing poolname parameter")
+    }
 
-      val poolToActiveStages = listener.poolToActiveStages
-      val activeStages = poolToActiveStages.get(poolName) match {
-        case Some(s) => s.values.toSeq
-        case None => Seq[StageInfo]()
-      }
-      val shouldShowActiveStages = activeStages.nonEmpty
-      val activeStagesTable =
-        new StageTableBase(request, activeStages, "", "activeStage", parent.basePath, "stages/pool",
-          parent.progressListener, parent.isFairScheduler, parent.killEnabled,
-          isFailedStage = false)
+    // For now, pool information is only accessible in live UIs
+    val pool = parent.sc.flatMap(_.getPoolForName(poolName)).getOrElse {
+      throw new IllegalArgumentException(s"Unknown pool: $poolName")
+    }
 
-      // For now, pool information is only accessible in live UIs
-      val pools = sc.map(_.getPoolForName(poolName).getOrElse {
-        throw new IllegalArgumentException(s"Unknown poolname: $poolName")
-      }).toSeq
-      val poolTable = new PoolTable(pools, parent)
+    val uiPool = try {
+      parent.store.pool(poolName)
+    } catch {
+      case _: NoSuchElementException =>
+        new PoolData(poolName, Set())
+    }
 
-      var content = <h4>Summary </h4> ++ poolTable.toNodeSeq
-      if (shouldShowActiveStages) {
-        content ++= <h4>{activeStages.size} Active Stages</h4> ++ activeStagesTable.toNodeSeq
-      }
+    val activeStages = uiPool.stageIds.toSeq.map(parent.store.lastStageAttempt(_))
 
-      UIUtils.headerSparkPage("Fair Scheduler Pool: " + poolName, content, parent)
+    val shouldShowActiveStages = activeStages.nonEmpty
+    val activeStagesTable =
+      new StageTableBase(parent.store, request, activeStages, "", "activeStage", parent.basePath,
+        "stages/pool", parent.isFairScheduler, parent.killEnabled, false)
+
+    val poolTable = new PoolTable(Map(pool -> uiPool), parent)
+    var content = <h4>Summary </h4> ++ poolTable.toNodeSeq
+    if (shouldShowActiveStages) {
+      content ++= <h4>{activeStages.size} Active Stages</h4> ++ activeStagesTable.toNodeSeq
     }
+
+    UIUtils.headerSparkPage("Fair Scheduler Pool: " + poolName, content, parent)
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/PoolTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/PoolTable.scala
index ea02968733cac..ddc42eea26704 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/PoolTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/PoolTable.scala
@@ -22,22 +22,15 @@ import java.net.URLEncoder
 import scala.collection.mutable.HashMap
 import scala.xml.Node
 
-import org.apache.spark.scheduler.{Schedulable, StageInfo}
+import org.apache.spark.scheduler.Schedulable
+import org.apache.spark.status.PoolData
+import org.apache.spark.status.api.v1.StageData
 import org.apache.spark.ui.UIUtils
 
 /** Table showing list of pools */
-private[ui] class PoolTable(pools: Seq[Schedulable], parent: StagesTab) {
-  private val listener = parent.progressListener
+private[ui] class PoolTable(pools: Map[Schedulable, PoolData], parent: StagesTab) {
 
   def toNodeSeq: Seq[Node] = {
-    listener.synchronized {
-      poolTable(poolRow, pools)
-    }
-  }
-
-  private def poolTable(
-      makeRow: (Schedulable, HashMap[String, HashMap[Int, StageInfo]]) => Seq[Node],
-      rows: Seq[Schedulable]): Seq[Node] = {
     <table class="table table-bordered table-striped table-condensed sortable table-fixed">
       <thead>
         <th>Pool Name</th>
@@ -48,29 +41,24 @@ private[ui] class PoolTable(pools: Seq[Schedulable], parent: StagesTab) {
         <th>SchedulingMode</th>
       </thead>
       <tbody>
-        {rows.map(r => makeRow(r, listener.poolToActiveStages))}
+        {pools.map { case (s, p) => poolRow(s, p) }}
       </tbody>
     </table>
   }
 
-  private def poolRow(
-      p: Schedulable,
-      poolToActiveStages: HashMap[String, HashMap[Int, StageInfo]]): Seq[Node] = {
-    val activeStages = poolToActiveStages.get(p.name) match {
-      case Some(stages) => stages.size
-      case None => 0
-    }
+  private def poolRow(s: Schedulable, p: PoolData): Seq[Node] = {
+    val activeStages = p.stageIds.size
     val href = "%s/stages/pool?poolname=%s"
       .format(UIUtils.prependBaseUri(parent.basePath), URLEncoder.encode(p.name, "UTF-8"))
     <tr>
       <td>
         <a href={href}>{p.name}</a>
       </td>
-      <td>{p.minShare}</td>
-      <td>{p.weight}</td>
+      <td>{s.minShare}</td>
+      <td>{s.weight}</td>
       <td>{activeStages}</td>
-      <td>{p.runningTasks}</td>
-      <td>{p.schedulingMode}</td>
+      <td>{s.runningTasks}</td>
+      <td>{s.schedulingMode}</td>
     </tr>
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index db79dd90f1659..9ea618e7e3086 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -18,7 +18,7 @@
 package org.apache.spark.ui.jobs
 
 import java.net.URLEncoder
-import java.util.Date
+import java.util.{Date, NoSuchElementException}
 import javax.servlet.http.HttpServletRequest
 
 import scala.collection.mutable.HashSet
@@ -27,10 +27,12 @@ import scala.xml.{Elem, Node, Unparsed}
 import org.apache.commons.lang3.StringEscapeUtils
 
 import org.apache.spark.SparkConf
-import org.apache.spark.executor.TaskMetrics
-import org.apache.spark.scheduler.{AccumulableInfo, TaskInfo, TaskLocality}
+import org.apache.spark.internal.config._
+import org.apache.spark.scheduler.{TaskInfo, TaskLocality}
 import org.apache.spark.status.AppStateStore
+import org.apache.spark.status.api.v1._
 import org.apache.spark.ui._
+import org.apache.spark.ui.jobs.ApiHelper._
 import org.apache.spark.ui.jobs.UIData._
 import org.apache.spark.util.{Distribution, Utils}
 
@@ -38,9 +40,6 @@ import org.apache.spark.util.{Distribution, Utils}
 private[ui] class StagePage(parent: StagesTab, store: AppStateStore) extends WebUIPage("stage") {
   import StagePage._
 
-  private val progressListener = parent.progressListener
-  private val operationGraphListener = parent.operationGraphListener
-
   private val TIMELINE_LEGEND = {
     <div class="legend-area">
       <svg>
@@ -69,553 +68,529 @@ private[ui] class StagePage(parent: StagesTab, store: AppStateStore) extends Web
   // if we find that it's okay.
   private val MAX_TIMELINE_TASKS = parent.conf.getInt("spark.ui.timeline.tasks.maximum", 1000)
 
-  private def getLocalitySummaryString(stageData: StageUIData): String = {
-    val localities = stageData.taskData.values.map(_.taskInfo.taskLocality)
+  private def getLocalitySummaryString(stageData: StageData, taskList: Seq[TaskData]): String = {
+    val localities = taskList.map(_.taskLocality)
     val localityCounts = localities.groupBy(identity).mapValues(_.size)
+    val names = Map(
+      TaskLocality.PROCESS_LOCAL.toString() -> "Process local",
+      TaskLocality.NODE_LOCAL.toString() -> "Node local",
+      TaskLocality.RACK_LOCAL.toString() -> "Rack local",
+      TaskLocality.ANY.toString() -> "Any")
     val localityNamesAndCounts = localityCounts.toSeq.map { case (locality, count) =>
-      val localityName = locality match {
-        case TaskLocality.PROCESS_LOCAL => "Process local"
-        case TaskLocality.NODE_LOCAL => "Node local"
-        case TaskLocality.RACK_LOCAL => "Rack local"
-        case TaskLocality.ANY => "Any"
-      }
-      s"$localityName: $count"
+      s"${names(locality)}: $count"
     }
     localityNamesAndCounts.sorted.mkString("; ")
   }
 
   def render(request: HttpServletRequest): Seq[Node] = {
-    progressListener.synchronized {
-      val parameterId = request.getParameter("id")
-      require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
-
-      val parameterAttempt = request.getParameter("attempt")
-      require(parameterAttempt != null && parameterAttempt.nonEmpty, "Missing attempt parameter")
-
-      val parameterTaskPage = request.getParameter("task.page")
-      val parameterTaskSortColumn = request.getParameter("task.sort")
-      val parameterTaskSortDesc = request.getParameter("task.desc")
-      val parameterTaskPageSize = request.getParameter("task.pageSize")
-      val parameterTaskPrevPageSize = request.getParameter("task.prevPageSize")
-
-      val taskPage = Option(parameterTaskPage).map(_.toInt).getOrElse(1)
-      val taskSortColumn = Option(parameterTaskSortColumn).map { sortColumn =>
-        UIUtils.decodeURLParameter(sortColumn)
-      }.getOrElse("Index")
-      val taskSortDesc = Option(parameterTaskSortDesc).map(_.toBoolean).getOrElse(false)
-      val taskPageSize = Option(parameterTaskPageSize).map(_.toInt).getOrElse(100)
-      val taskPrevPageSize = Option(parameterTaskPrevPageSize).map(_.toInt).getOrElse(taskPageSize)
-
-      val stageId = parameterId.toInt
-      val stageAttemptId = parameterAttempt.toInt
-      val stageDataOption = progressListener.stageIdToData.get((stageId, stageAttemptId))
-
-      val stageHeader = s"Details for Stage $stageId (Attempt $stageAttemptId)"
-      if (stageDataOption.isEmpty) {
+    val parameterId = request.getParameter("id")
+    require(parameterId != null && parameterId.nonEmpty, "Missing id parameter")
+
+    val parameterAttempt = request.getParameter("attempt")
+    require(parameterAttempt != null && parameterAttempt.nonEmpty, "Missing attempt parameter")
+
+    val parameterTaskPage = request.getParameter("task.page")
+    val parameterTaskSortColumn = request.getParameter("task.sort")
+    val parameterTaskSortDesc = request.getParameter("task.desc")
+    val parameterTaskPageSize = request.getParameter("task.pageSize")
+    val parameterTaskPrevPageSize = request.getParameter("task.prevPageSize")
+
+    val taskPage = Option(parameterTaskPage).map(_.toInt).getOrElse(1)
+    val taskSortColumn = Option(parameterTaskSortColumn).map { sortColumn =>
+      UIUtils.decodeURLParameter(sortColumn)
+    }.getOrElse("Index")
+    val taskSortDesc = Option(parameterTaskSortDesc).map(_.toBoolean).getOrElse(false)
+    val taskPageSize = Option(parameterTaskPageSize).map(_.toInt).getOrElse(100)
+    val taskPrevPageSize = Option(parameterTaskPrevPageSize).map(_.toInt).getOrElse(taskPageSize)
+
+    val stageId = parameterId.toInt
+    val stageAttemptId = parameterAttempt.toInt
+
+    val stageHeader = s"Details for Stage $stageId (Attempt $stageAttemptId)"
+    val stageData = try {
+      parent.store.stageAttempt(stageId, stageAttemptId)
+    } catch {
+      case _: NoSuchElementException =>
         val content =
           <div id="no-info">
             <p>No information to display for Stage {stageId} (Attempt {stageAttemptId})</p>
           </div>
         return UIUtils.headerSparkPage(stageHeader, content, parent)
+    }
 
-      }
-      if (stageDataOption.get.taskData.isEmpty) {
-        val content =
-          <div>
-            <h4>Summary Metrics</h4> No tasks have started yet
-            <h4>Tasks</h4> No tasks have started yet
-          </div>
-        return UIUtils.headerSparkPage(stageHeader, content, parent)
-      }
+    // Retrieve at most the configured max tasks from the store, most recent first.
+    val maxTasks = parent.conf.get(UI_RETAINED_TASKS)
+    val taskList = parent.store.taskList(stageId, stageAttemptId, maxTasks)
 
-      val stageData = stageDataOption.get
-      val tasks = stageData.taskData.values.toSeq.sortBy(_.taskInfo.launchTime)
-      val numCompleted = stageData.numCompleteTasks
-      val totalTasks = stageData.numActiveTasks +
-        stageData.numCompleteTasks + stageData.numFailedTasks
-      val totalTasksNumStr = if (totalTasks == tasks.size) {
-        s"$totalTasks"
-      } else {
-        s"$totalTasks, showing ${tasks.size}"
-      }
+    if (taskList.isEmpty) {
+      val content =
+        <div>
+          <h4>Summary Metrics</h4> No tasks have started yet
+          <h4>Tasks</h4> No tasks have started yet
+        </div>
+      return UIUtils.headerSparkPage(stageHeader, content, parent)
+    }
 
-      val allAccumulables = progressListener.stageIdToData((stageId, stageAttemptId)).accumulables
-      val externalAccumulables = allAccumulables.values.filter { acc => !acc.internal }
-      val hasAccumulators = externalAccumulables.nonEmpty
+    val tasks = taskList.sortBy(_.launchTime)
+    val numCompleted = stageData.numCompleteTasks
+    val totalTasks = stageData.numActiveTasks +
+      stageData.numCompleteTasks + stageData.numFailedTasks
+    val totalTasksNumStr = if (totalTasks == tasks.size) {
+      s"$totalTasks"
+    } else {
+      s"$totalTasks, showing ${tasks.size}"
+    }
 
-      val summary =
-        <div>
-          <ul class="unstyled">
+    val externalAccumulables = stageData.accumulatorUpdates
+    val hasAccumulators = externalAccumulables.size > 0
+
+    val summary =
+      <div>
+        <ul class="unstyled">
+          <li>
+            <strong>Total Time Across All Tasks: </strong>
+            {UIUtils.formatDuration(stageData.executorRunTime)}
+          </li>
+          <li>
+            <strong>Locality Level Summary: </strong>
+            {getLocalitySummaryString(stageData, taskList)}
+          </li>
+          {if (hasInput(stageData)) {
             <li>
-              <strong>Total Time Across All Tasks: </strong>
-              {UIUtils.formatDuration(stageData.executorRunTime)}
+              <strong>Input Size / Records: </strong>
+              {s"${Utils.bytesToString(stageData.inputBytes)} / ${stageData.inputRecords}"}
             </li>
+          }}
+          {if (hasOutput(stageData)) {
             <li>
-              <strong>Locality Level Summary: </strong>
-              {getLocalitySummaryString(stageData)}
+              <strong>Output: </strong>
+              {s"${Utils.bytesToString(stageData.outputBytes)} / ${stageData.outputRecords}"}
             </li>
-            {if (stageData.hasInput) {
-              <li>
-                <strong>Input Size / Records: </strong>
-                {s"${Utils.bytesToString(stageData.inputBytes)} / ${stageData.inputRecords}"}
-              </li>
-            }}
-            {if (stageData.hasOutput) {
-              <li>
-                <strong>Output: </strong>
-                {s"${Utils.bytesToString(stageData.outputBytes)} / ${stageData.outputRecords}"}
-              </li>
-            }}
-            {if (stageData.hasShuffleRead) {
-              <li>
-                <strong>Shuffle Read: </strong>
-                {s"${Utils.bytesToString(stageData.shuffleReadTotalBytes)} / " +
-                 s"${stageData.shuffleReadRecords}"}
-              </li>
-            }}
-            {if (stageData.hasShuffleWrite) {
-              <li>
-                <strong>Shuffle Write: </strong>
-                 {s"${Utils.bytesToString(stageData.shuffleWriteBytes)} / " +
-                 s"${stageData.shuffleWriteRecords}"}
-              </li>
-            }}
-            {if (stageData.hasBytesSpilled) {
-              <li>
-                <strong>Shuffle Spill (Memory): </strong>
-                {Utils.bytesToString(stageData.memoryBytesSpilled)}
-              </li>
-              <li>
-                <strong>Shuffle Spill (Disk): </strong>
-                {Utils.bytesToString(stageData.diskBytesSpilled)}
-              </li>
-            }}
-          </ul>
-        </div>
+          }}
+          {if (hasShuffleRead(stageData)) {
+            <li>
+              <strong>Shuffle Read: </strong>
+              {s"${Utils.bytesToString(stageData.shuffleReadBytes)} / " +
+               s"${stageData.shuffleReadRecords}"}
+            </li>
+          }}
+          {if (hasShuffleWrite(stageData)) {
+            <li>
+              <strong>Shuffle Write: </strong>
+               {s"${Utils.bytesToString(stageData.shuffleWriteBytes)} / " +
+               s"${stageData.shuffleWriteRecords}"}
+            </li>
+          }}
+          {if (hasBytesSpilled(stageData)) {
+            <li>
+              <strong>Shuffle Spill (Memory): </strong>
+              {Utils.bytesToString(stageData.memoryBytesSpilled)}
+            </li>
+            <li>
+              <strong>Shuffle Spill (Disk): </strong>
+              {Utils.bytesToString(stageData.diskBytesSpilled)}
+            </li>
+          }}
+        </ul>
+      </div>
 
-      val showAdditionalMetrics =
-        <div>
-          <span class="expand-additional-metrics">
-            <span class="expand-additional-metrics-arrow arrow-closed"></span>
-            <a>Show Additional Metrics</a>
-          </span>
-          <div class="additional-metrics collapsed">
-            <ul>
-              <li>
-                  <input type="checkbox" id="select-all-metrics"/>
-                  <span class="additional-metric-title"><em>(De)select All</em></span>
-              </li>
-              <li>
-                <span data-toggle="tooltip"
-                      title={ToolTips.SCHEDULER_DELAY} data-placement="right">
-                  <input type="checkbox" name={TaskDetailsClassNames.SCHEDULER_DELAY}/>
-                  <span class="additional-metric-title">Scheduler Delay</span>
-                </span>
-              </li>
-              <li>
-                <span data-toggle="tooltip"
-                      title={ToolTips.TASK_DESERIALIZATION_TIME} data-placement="right">
-                  <input type="checkbox" name={TaskDetailsClassNames.TASK_DESERIALIZATION_TIME}/>
-                  <span class="additional-metric-title">Task Deserialization Time</span>
-                </span>
-              </li>
-              {if (stageData.hasShuffleRead) {
-                <li>
-                  <span data-toggle="tooltip"
-                        title={ToolTips.SHUFFLE_READ_BLOCKED_TIME} data-placement="right">
-                    <input type="checkbox" name={TaskDetailsClassNames.SHUFFLE_READ_BLOCKED_TIME}/>
-                    <span class="additional-metric-title">Shuffle Read Blocked Time</span>
-                  </span>
-                </li>
-                <li>
-                  <span data-toggle="tooltip"
-                        title={ToolTips.SHUFFLE_READ_REMOTE_SIZE} data-placement="right">
-                    <input type="checkbox" name={TaskDetailsClassNames.SHUFFLE_READ_REMOTE_SIZE}/>
-                    <span class="additional-metric-title">Shuffle Remote Reads</span>
-                  </span>
-                </li>
-              }}
-              <li>
-                <span data-toggle="tooltip"
-                      title={ToolTips.RESULT_SERIALIZATION_TIME} data-placement="right">
-                  <input type="checkbox" name={TaskDetailsClassNames.RESULT_SERIALIZATION_TIME}/>
-                  <span class="additional-metric-title">Result Serialization Time</span>
-                </span>
-              </li>
+    val showAdditionalMetrics =
+      <div>
+        <span class="expand-additional-metrics">
+          <span class="expand-additional-metrics-arrow arrow-closed"></span>
+          <a>Show Additional Metrics</a>
+        </span>
+        <div class="additional-metrics collapsed">
+          <ul>
+            <li>
+                <input type="checkbox" id="select-all-metrics"/>
+                <span class="additional-metric-title"><em>(De)select All</em></span>
+            </li>
+            <li>
+              <span data-toggle="tooltip"
+                    title={ToolTips.SCHEDULER_DELAY} data-placement="right">
+                <input type="checkbox" name={TaskDetailsClassNames.SCHEDULER_DELAY}/>
+                <span class="additional-metric-title">Scheduler Delay</span>
+              </span>
+            </li>
+            <li>
+              <span data-toggle="tooltip"
+                    title={ToolTips.TASK_DESERIALIZATION_TIME} data-placement="right">
+                <input type="checkbox" name={TaskDetailsClassNames.TASK_DESERIALIZATION_TIME}/>
+                <span class="additional-metric-title">Task Deserialization Time</span>
+              </span>
+            </li>
+            {if (stageData.shuffleReadBytes > 0) {
               <li>
                 <span data-toggle="tooltip"
-                      title={ToolTips.GETTING_RESULT_TIME} data-placement="right">
-                  <input type="checkbox" name={TaskDetailsClassNames.GETTING_RESULT_TIME}/>
-                  <span class="additional-metric-title">Getting Result Time</span>
+                      title={ToolTips.SHUFFLE_READ_BLOCKED_TIME} data-placement="right">
+                  <input type="checkbox" name={TaskDetailsClassNames.SHUFFLE_READ_BLOCKED_TIME}/>
+                  <span class="additional-metric-title">Shuffle Read Blocked Time</span>
                 </span>
               </li>
               <li>
                 <span data-toggle="tooltip"
-                      title={ToolTips.PEAK_EXECUTION_MEMORY} data-placement="right">
-                  <input type="checkbox" name={TaskDetailsClassNames.PEAK_EXECUTION_MEMORY}/>
-                  <span class="additional-metric-title">Peak Execution Memory</span>
+                      title={ToolTips.SHUFFLE_READ_REMOTE_SIZE} data-placement="right">
+                  <input type="checkbox" name={TaskDetailsClassNames.SHUFFLE_READ_REMOTE_SIZE}/>
+                  <span class="additional-metric-title">Shuffle Remote Reads</span>
                 </span>
               </li>
-            </ul>
-          </div>
-        </div>
-
-      val dagViz = UIUtils.showDagVizForStage(
-        stageId, operationGraphListener.getOperationGraphForStage(stageId))
-
-      val accumulableHeaders: Seq[String] = Seq("Accumulable", "Value")
-      def accumulableRow(acc: AccumulableInfo): Seq[Node] = {
-        (acc.name, acc.value) match {
-          case (Some(name), Some(value)) => <tr><td>{name}</td><td>{value}</td></tr>
-          case _ => Seq.empty[Node]
-        }
-      }
-      val accumulableTable = UIUtils.listingTable(
-        accumulableHeaders,
-        accumulableRow,
-        externalAccumulables.toSeq)
-
-      val page: Int = {
-        // If the user has changed to a larger page size, then go to page 1 in order to avoid
-        // IndexOutOfBoundsException.
-        if (taskPageSize <= taskPrevPageSize) {
-          taskPage
-        } else {
-          1
-        }
-      }
-      val currentTime = System.currentTimeMillis()
-      val (taskTable, taskTableHTML) = try {
-        val _taskTable = new TaskPagedTable(
-          parent.conf,
-          UIUtils.prependBaseUri(parent.basePath) +
-            s"/stages/stage?id=${stageId}&attempt=${stageAttemptId}",
-          tasks,
-          hasAccumulators,
-          stageData.hasInput,
-          stageData.hasOutput,
-          stageData.hasShuffleRead,
-          stageData.hasShuffleWrite,
-          stageData.hasBytesSpilled,
-          currentTime,
-          pageSize = taskPageSize,
-          sortColumn = taskSortColumn,
-          desc = taskSortDesc,
-          store = store
-        )
-        (_taskTable, _taskTable.table(page))
-      } catch {
-        case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
-          val errorMessage =
-            <div class="alert alert-error">
-              <p>Error while rendering stage table:</p>
-              <pre>
-                {Utils.exceptionString(e)}
-              </pre>
-            </div>
-          (null, errorMessage)
-      }
-
-      val jsForScrollingDownToTaskTable =
-        <script>
-          {Unparsed {
-            """
-              |$(function() {
-              |  if (/.*&task.sort=.*$/.test(location.search)) {
-              |    var topOffset = $("#tasks-section").offset().top;
-              |    $("html,body").animate({scrollTop: topOffset}, 200);
-              |  }
-              |});
-            """.stripMargin
-           }
-          }
-        </script>
-
-      val taskIdsInPage = if (taskTable == null) Set.empty[Long]
-        else taskTable.dataSource.slicedTaskIds
-
-      // Excludes tasks which failed and have incomplete metrics
-      val validTasks = tasks.filter(t => t.taskInfo.status == "SUCCESS" && t.metrics.isDefined)
-
-      val summaryTable: Option[Seq[Node]] =
-        if (validTasks.isEmpty) {
-          None
-        }
-        else {
-          def getDistributionQuantiles(data: Seq[Double]): IndexedSeq[Double] =
-            Distribution(data).get.getQuantiles()
-          def getFormattedTimeQuantiles(times: Seq[Double]): Seq[Node] = {
-            getDistributionQuantiles(times).map { millis =>
-              <td>{UIUtils.formatDuration(millis.toLong)}</td>
-            }
-          }
-          def getFormattedSizeQuantiles(data: Seq[Double]): Seq[Elem] = {
-            getDistributionQuantiles(data).map(d => <td>{Utils.bytesToString(d.toLong)}</td>)
-          }
-
-          val deserializationTimes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.executorDeserializeTime.toDouble
-          }
-          val deserializationQuantiles =
-            <td>
-              <span data-toggle="tooltip" title={ToolTips.TASK_DESERIALIZATION_TIME}
-                    data-placement="right">
-                Task Deserialization Time
-              </span>
-            </td> +: getFormattedTimeQuantiles(deserializationTimes)
-
-          val serviceTimes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.executorRunTime.toDouble
-          }
-          val serviceQuantiles = <td>Duration</td> +: getFormattedTimeQuantiles(serviceTimes)
-
-          val gcTimes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.jvmGCTime.toDouble
-          }
-          val gcQuantiles =
-            <td>
-              <span data-toggle="tooltip"
-                  title={ToolTips.GC_TIME} data-placement="right">GC Time
-              </span>
-            </td> +: getFormattedTimeQuantiles(gcTimes)
-
-          val serializationTimes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.resultSerializationTime.toDouble
-          }
-          val serializationQuantiles =
-            <td>
+            }}
+            <li>
               <span data-toggle="tooltip"
                     title={ToolTips.RESULT_SERIALIZATION_TIME} data-placement="right">
-                Result Serialization Time
+                <input type="checkbox" name={TaskDetailsClassNames.RESULT_SERIALIZATION_TIME}/>
+                <span class="additional-metric-title">Result Serialization Time</span>
               </span>
-            </td> +: getFormattedTimeQuantiles(serializationTimes)
-
-          val gettingResultTimes = validTasks.map { taskUIData: TaskUIData =>
-            getGettingResultTime(taskUIData.taskInfo, currentTime).toDouble
-          }
-          val gettingResultQuantiles =
-            <td>
+            </li>
+            <li>
               <span data-toggle="tooltip"
-                  title={ToolTips.GETTING_RESULT_TIME} data-placement="right">
-                Getting Result Time
+                    title={ToolTips.GETTING_RESULT_TIME} data-placement="right">
+                <input type="checkbox" name={TaskDetailsClassNames.GETTING_RESULT_TIME}/>
+                <span class="additional-metric-title">Getting Result Time</span>
               </span>
-            </td> +:
-            getFormattedTimeQuantiles(gettingResultTimes)
-
-          val peakExecutionMemory = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.peakExecutionMemory.toDouble
-          }
-          val peakExecutionMemoryQuantiles = {
-            <td>
+            </li>
+            <li>
               <span data-toggle="tooltip"
                     title={ToolTips.PEAK_EXECUTION_MEMORY} data-placement="right">
-                Peak Execution Memory
+                <input type="checkbox" name={TaskDetailsClassNames.PEAK_EXECUTION_MEMORY}/>
+                <span class="additional-metric-title">Peak Execution Memory</span>
               </span>
-            </td> +: getFormattedSizeQuantiles(peakExecutionMemory)
-          }
-
-          // The scheduler delay includes the network delay to send the task to the worker
-          // machine and to send back the result (but not the time to fetch the task result,
-          // if it needed to be fetched from the block manager on the worker).
-          val schedulerDelays = validTasks.map { taskUIData: TaskUIData =>
-            getSchedulerDelay(taskUIData.taskInfo, taskUIData.metrics.get, currentTime).toDouble
-          }
-          val schedulerDelayTitle = <td><span data-toggle="tooltip"
-            title={ToolTips.SCHEDULER_DELAY} data-placement="right">Scheduler Delay</span></td>
-          val schedulerDelayQuantiles = schedulerDelayTitle +:
-            getFormattedTimeQuantiles(schedulerDelays)
-          def getFormattedSizeQuantilesWithRecords(data: Seq[Double], records: Seq[Double])
-            : Seq[Elem] = {
-            val recordDist = getDistributionQuantiles(records).iterator
-            getDistributionQuantiles(data).map(d =>
-              <td>{s"${Utils.bytesToString(d.toLong)} / ${recordDist.next().toLong}"}</td>
-            )
-          }
-
-          val inputSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.inputMetrics.bytesRead.toDouble
-          }
-
-          val inputRecords = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.inputMetrics.recordsRead.toDouble
-          }
-
-          val inputQuantiles = <td>Input Size / Records</td> +:
-            getFormattedSizeQuantilesWithRecords(inputSizes, inputRecords)
+            </li>
+          </ul>
+        </div>
+      </div>
 
-          val outputSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.outputMetrics.bytesWritten.toDouble
-          }
+    val stageGraph = try {
+      Some(parent.store.operationGraphForStage(stageId))
+    } catch {
+      case _: NoSuchElementException => None
+    }
+    val dagViz = UIUtils.showDagVizForStage(stageId, stageGraph)
 
-          val outputRecords = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.outputMetrics.recordsWritten.toDouble
-          }
+    val accumulableHeaders: Seq[String] = Seq("Accumulable", "Value")
+    def accumulableRow(acc: AccumulableInfo): Seq[Node] = {
+      <tr><td>{acc.name}</td><td>{acc.value}</td></tr>
+    }
+    val accumulableTable = UIUtils.listingTable(
+      accumulableHeaders,
+      accumulableRow,
+      externalAccumulables.toSeq)
+
+    val page: Int = {
+      // If the user has changed to a larger page size, then go to page 1 in order to avoid
+      // IndexOutOfBoundsException.
+      if (taskPageSize <= taskPrevPageSize) {
+        taskPage
+      } else {
+        1
+      }
+    }
+    val currentTime = System.currentTimeMillis()
+    val (taskTable, taskTableHTML) = try {
+      val _taskTable = new TaskPagedTable(
+        parent.conf,
+        UIUtils.prependBaseUri(parent.basePath) +
+          s"/stages/stage?id=${stageId}&attempt=${stageAttemptId}",
+        tasks,
+        hasAccumulators,
+        hasInput(stageData),
+        hasOutput(stageData),
+        hasShuffleRead(stageData),
+        hasShuffleWrite(stageData),
+        hasBytesSpilled(stageData),
+        currentTime,
+        pageSize = taskPageSize,
+        sortColumn = taskSortColumn,
+        desc = taskSortDesc,
+        store = parent.store
+      )
+      (_taskTable, _taskTable.table(page))
+    } catch {
+      case e @ (_ : IllegalArgumentException | _ : IndexOutOfBoundsException) =>
+        val errorMessage =
+          <div class="alert alert-error">
+            <p>Error while rendering stage table:</p>
+            <pre>
+              {Utils.exceptionString(e)}
+            </pre>
+          </div>
+        (null, errorMessage)
+    }
 
-          val outputQuantiles = <td>Output Size / Records</td> +:
-            getFormattedSizeQuantilesWithRecords(outputSizes, outputRecords)
+    val jsForScrollingDownToTaskTable =
+      <script>
+        {Unparsed {
+          """
+            |$(function() {
+            |  if (/.*&task.sort=.*$/.test(location.search)) {
+            |    var topOffset = $("#tasks-section").offset().top;
+            |    $("html,body").animate({scrollTop: topOffset}, 200);
+            |  }
+            |});
+          """.stripMargin
+         }
+        }
+      </script>
 
-          val shuffleReadBlockedTimes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.shuffleReadMetrics.fetchWaitTime.toDouble
-          }
-          val shuffleReadBlockedQuantiles =
-            <td>
-              <span data-toggle="tooltip"
-                    title={ToolTips.SHUFFLE_READ_BLOCKED_TIME} data-placement="right">
-                Shuffle Read Blocked Time
-              </span>
-            </td> +:
-            getFormattedTimeQuantiles(shuffleReadBlockedTimes)
+    val taskIdsInPage = if (taskTable == null) Set.empty[Long]
+      else taskTable.dataSource.slicedTaskIds
 
-          val shuffleReadTotalSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.shuffleReadMetrics.totalBytesRead.toDouble
-          }
-          val shuffleReadTotalRecords = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.shuffleReadMetrics.recordsRead.toDouble
-          }
-          val shuffleReadTotalQuantiles =
-            <td>
-              <span data-toggle="tooltip"
-                    title={ToolTips.SHUFFLE_READ} data-placement="right">
-                Shuffle Read Size / Records
-              </span>
-            </td> +:
-            getFormattedSizeQuantilesWithRecords(shuffleReadTotalSizes, shuffleReadTotalRecords)
+    // Excludes tasks which failed and have incomplete metrics
+    val validTasks = tasks.filter(t => t.status == "SUCCESS" && t.taskMetrics.isDefined)
 
-          val shuffleReadRemoteSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.shuffleReadMetrics.remoteBytesRead.toDouble
+    val summaryTable: Option[Seq[Node]] =
+      if (validTasks.size == 0) {
+        None
+      }
+      else {
+        def getDistributionQuantiles(data: Seq[Double]): IndexedSeq[Double] =
+          Distribution(data).get.getQuantiles()
+        def getFormattedTimeQuantiles(times: Seq[Double]): Seq[Node] = {
+          getDistributionQuantiles(times).map { millis =>
+            <td>{UIUtils.formatDuration(millis.toLong)}</td>
           }
-          val shuffleReadRemoteQuantiles =
-            <td>
-              <span data-toggle="tooltip"
-                    title={ToolTips.SHUFFLE_READ_REMOTE_SIZE} data-placement="right">
-                Shuffle Remote Reads
-              </span>
-            </td> +:
-            getFormattedSizeQuantiles(shuffleReadRemoteSizes)
+        }
+        def getFormattedSizeQuantiles(data: Seq[Double]): Seq[Elem] = {
+          getDistributionQuantiles(data).map(d => <td>{Utils.bytesToString(d.toLong)}</td>)
+        }
 
-          val shuffleWriteSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.shuffleWriteMetrics.bytesWritten.toDouble
-          }
+        val deserializationTimes = validTasks.map { task =>
+          task.taskMetrics.get.executorDeserializeTime.toDouble
+        }
+        val deserializationQuantiles =
+          <td>
+            <span data-toggle="tooltip" title={ToolTips.TASK_DESERIALIZATION_TIME}
+                  data-placement="right">
+              Task Deserialization Time
+            </span>
+          </td> +: getFormattedTimeQuantiles(deserializationTimes)
+
+        val serviceTimes = validTasks.map(_.taskMetrics.get.executorRunTime.toDouble)
+        val serviceQuantiles = <td>Duration</td> +: getFormattedTimeQuantiles(serviceTimes)
+
+        val gcTimes = validTasks.map(_.taskMetrics.get.jvmGcTime.toDouble)
+        val gcQuantiles =
+          <td>
+            <span data-toggle="tooltip"
+                title={ToolTips.GC_TIME} data-placement="right">GC Time
+            </span>
+          </td> +: getFormattedTimeQuantiles(gcTimes)
+
+        val serializationTimes = validTasks.map(_.taskMetrics.get.resultSerializationTime.toDouble)
+        val serializationQuantiles =
+          <td>
+            <span data-toggle="tooltip"
+                  title={ToolTips.RESULT_SERIALIZATION_TIME} data-placement="right">
+              Result Serialization Time
+            </span>
+          </td> +: getFormattedTimeQuantiles(serializationTimes)
+
+        val gettingResultTimes = validTasks.map(getGettingResultTime(_, currentTime).toDouble)
+        val gettingResultQuantiles =
+          <td>
+            <span data-toggle="tooltip"
+                title={ToolTips.GETTING_RESULT_TIME} data-placement="right">
+              Getting Result Time
+            </span>
+          </td> +:
+          getFormattedTimeQuantiles(gettingResultTimes)
+
+        val peakExecutionMemory = validTasks.map(_.taskMetrics.get.peakExecutionMemory.toDouble)
+        val peakExecutionMemoryQuantiles = {
+          <td>
+            <span data-toggle="tooltip"
+                  title={ToolTips.PEAK_EXECUTION_MEMORY} data-placement="right">
+              Peak Execution Memory
+            </span>
+          </td> +: getFormattedSizeQuantiles(peakExecutionMemory)
+        }
 
-          val shuffleWriteRecords = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.shuffleWriteMetrics.recordsWritten.toDouble
-          }
+        // The scheduler delay includes the network delay to send the task to the worker
+        // machine and to send back the result (but not the time to fetch the task result,
+        // if it needed to be fetched from the block manager on the worker).
+        val schedulerDelays = validTasks.map { task =>
+          getSchedulerDelay(task, task.taskMetrics.get, currentTime).toDouble
+        }
+        val schedulerDelayTitle = <td><span data-toggle="tooltip"
+          title={ToolTips.SCHEDULER_DELAY} data-placement="right">Scheduler Delay</span></td>
+        val schedulerDelayQuantiles = schedulerDelayTitle +:
+          getFormattedTimeQuantiles(schedulerDelays)
+        def getFormattedSizeQuantilesWithRecords(data: Seq[Double], records: Seq[Double])
+          : Seq[Elem] = {
+          val recordDist = getDistributionQuantiles(records).iterator
+          getDistributionQuantiles(data).map(d =>
+            <td>{s"${Utils.bytesToString(d.toLong)} / ${recordDist.next().toLong}"}</td>
+          )
+        }
 
-          val shuffleWriteQuantiles = <td>Shuffle Write Size / Records</td> +:
-            getFormattedSizeQuantilesWithRecords(shuffleWriteSizes, shuffleWriteRecords)
+        val inputSizes = validTasks.map(_.taskMetrics.get.inputMetrics.bytesRead.toDouble)
+        val inputRecords = validTasks.map(_.taskMetrics.get.inputMetrics.recordsRead.toDouble)
+        val inputQuantiles = <td>Input Size / Records</td> +:
+          getFormattedSizeQuantilesWithRecords(inputSizes, inputRecords)
 
-          val memoryBytesSpilledSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.memoryBytesSpilled.toDouble
-          }
-          val memoryBytesSpilledQuantiles = <td>Shuffle spill (memory)</td> +:
-            getFormattedSizeQuantiles(memoryBytesSpilledSizes)
+        val outputSizes = validTasks.map(_.taskMetrics.get.outputMetrics.bytesWritten.toDouble)
+        val outputRecords = validTasks.map(_.taskMetrics.get.outputMetrics.recordsWritten.toDouble)
+        val outputQuantiles = <td>Output Size / Records</td> +:
+          getFormattedSizeQuantilesWithRecords(outputSizes, outputRecords)
 
-          val diskBytesSpilledSizes = validTasks.map { taskUIData: TaskUIData =>
-            taskUIData.metrics.get.diskBytesSpilled.toDouble
-          }
-          val diskBytesSpilledQuantiles = <td>Shuffle spill (disk)</td> +:
-            getFormattedSizeQuantiles(diskBytesSpilledSizes)
-
-          val listings: Seq[Seq[Node]] = Seq(
-            <tr>{serviceQuantiles}</tr>,
-            <tr class={TaskDetailsClassNames.SCHEDULER_DELAY}>{schedulerDelayQuantiles}</tr>,
-            <tr class={TaskDetailsClassNames.TASK_DESERIALIZATION_TIME}>
-              {deserializationQuantiles}
-            </tr>
-            <tr>{gcQuantiles}</tr>,
-            <tr class={TaskDetailsClassNames.RESULT_SERIALIZATION_TIME}>
-              {serializationQuantiles}
-            </tr>,
-            <tr class={TaskDetailsClassNames.GETTING_RESULT_TIME}>{gettingResultQuantiles}</tr>,
-            <tr class={TaskDetailsClassNames.PEAK_EXECUTION_MEMORY}>
-              {peakExecutionMemoryQuantiles}
-            </tr>,
-            if (stageData.hasInput) <tr>{inputQuantiles}</tr> else Nil,
-            if (stageData.hasOutput) <tr>{outputQuantiles}</tr> else Nil,
-            if (stageData.hasShuffleRead) {
-              <tr class={TaskDetailsClassNames.SHUFFLE_READ_BLOCKED_TIME}>
-                {shuffleReadBlockedQuantiles}
-              </tr>
-              <tr>{shuffleReadTotalQuantiles}</tr>
-              <tr class={TaskDetailsClassNames.SHUFFLE_READ_REMOTE_SIZE}>
-                {shuffleReadRemoteQuantiles}
-              </tr>
-            } else {
-              Nil
-            },
-            if (stageData.hasShuffleWrite) <tr>{shuffleWriteQuantiles}</tr> else Nil,
-            if (stageData.hasBytesSpilled) <tr>{memoryBytesSpilledQuantiles}</tr> else Nil,
-            if (stageData.hasBytesSpilled) <tr>{diskBytesSpilledQuantiles}</tr> else Nil)
-
-          val quantileHeaders = Seq("Metric", "Min", "25th percentile",
-            "Median", "75th percentile", "Max")
-          // The summary table does not use CSS to stripe rows, which doesn't work with hidden
-          // rows (instead, JavaScript in table.js is used to stripe the non-hidden rows).
-          Some(UIUtils.listingTable(
-            quantileHeaders,
-            identity[Seq[Node]],
-            listings,
-            fixedWidth = true,
-            id = Some("task-summary-table"),
-            stripeRowsWithCss = false))
+        val shuffleReadBlockedTimes = validTasks.map { task =>
+          task.taskMetrics.get.shuffleReadMetrics.fetchWaitTime.toDouble
+        }
+        val shuffleReadBlockedQuantiles =
+          <td>
+            <span data-toggle="tooltip"
+                  title={ToolTips.SHUFFLE_READ_BLOCKED_TIME} data-placement="right">
+              Shuffle Read Blocked Time
+            </span>
+          </td> +:
+          getFormattedTimeQuantiles(shuffleReadBlockedTimes)
+
+        val shuffleReadTotalSizes = validTasks.map { task =>
+          totalBytesRead(task.taskMetrics.get.shuffleReadMetrics).toDouble
+        }
+        val shuffleReadTotalRecords = validTasks.map { task =>
+          task.taskMetrics.get.shuffleReadMetrics.recordsRead.toDouble
+        }
+        val shuffleReadTotalQuantiles =
+          <td>
+            <span data-toggle="tooltip"
+                  title={ToolTips.SHUFFLE_READ} data-placement="right">
+              Shuffle Read Size / Records
+            </span>
+          </td> +:
+          getFormattedSizeQuantilesWithRecords(shuffleReadTotalSizes, shuffleReadTotalRecords)
+
+        val shuffleReadRemoteSizes = validTasks.map { task =>
+          task.taskMetrics.get.shuffleReadMetrics.remoteBytesRead.toDouble
+        }
+        val shuffleReadRemoteQuantiles =
+          <td>
+            <span data-toggle="tooltip"
+                  title={ToolTips.SHUFFLE_READ_REMOTE_SIZE} data-placement="right">
+              Shuffle Remote Reads
+            </span>
+          </td> +:
+          getFormattedSizeQuantiles(shuffleReadRemoteSizes)
+
+        val shuffleWriteSizes = validTasks.map { task =>
+          task.taskMetrics.get.shuffleWriteMetrics.bytesWritten.toDouble
         }
 
-      val executorTable = new ExecutorTable(stageId, stageAttemptId, parent, store)
+        val shuffleWriteRecords = validTasks.map { task =>
+          task.taskMetrics.get.shuffleWriteMetrics.recordsWritten.toDouble
+        }
 
-      val maybeAccumulableTable: Seq[Node] =
-        if (hasAccumulators) { <h4>Accumulators</h4> ++ accumulableTable } else Seq()
+        val shuffleWriteQuantiles = <td>Shuffle Write Size / Records</td> +:
+          getFormattedSizeQuantilesWithRecords(shuffleWriteSizes, shuffleWriteRecords)
+
+        val memoryBytesSpilledSizes = validTasks.map(_.taskMetrics.get.memoryBytesSpilled.toDouble)
+        val memoryBytesSpilledQuantiles = <td>Shuffle spill (memory)</td> +:
+          getFormattedSizeQuantiles(memoryBytesSpilledSizes)
+
+        val diskBytesSpilledSizes = validTasks.map(_.taskMetrics.get.diskBytesSpilled.toDouble)
+        val diskBytesSpilledQuantiles = <td>Shuffle spill (disk)</td> +:
+          getFormattedSizeQuantiles(diskBytesSpilledSizes)
+
+        val listings: Seq[Seq[Node]] = Seq(
+          <tr>{serviceQuantiles}</tr>,
+          <tr class={TaskDetailsClassNames.SCHEDULER_DELAY}>{schedulerDelayQuantiles}</tr>,
+          <tr class={TaskDetailsClassNames.TASK_DESERIALIZATION_TIME}>
+            {deserializationQuantiles}
+          </tr>
+          <tr>{gcQuantiles}</tr>,
+          <tr class={TaskDetailsClassNames.RESULT_SERIALIZATION_TIME}>
+            {serializationQuantiles}
+          </tr>,
+          <tr class={TaskDetailsClassNames.GETTING_RESULT_TIME}>{gettingResultQuantiles}</tr>,
+          <tr class={TaskDetailsClassNames.PEAK_EXECUTION_MEMORY}>
+            {peakExecutionMemoryQuantiles}
+          </tr>,
+          if (hasInput(stageData)) <tr>{inputQuantiles}</tr> else Nil,
+          if (hasOutput(stageData)) <tr>{outputQuantiles}</tr> else Nil,
+          if (hasShuffleRead(stageData)) {
+            <tr class={TaskDetailsClassNames.SHUFFLE_READ_BLOCKED_TIME}>
+              {shuffleReadBlockedQuantiles}
+            </tr>
+            <tr>{shuffleReadTotalQuantiles}</tr>
+            <tr class={TaskDetailsClassNames.SHUFFLE_READ_REMOTE_SIZE}>
+              {shuffleReadRemoteQuantiles}
+            </tr>
+          } else {
+            Nil
+          },
+          if (hasShuffleWrite(stageData)) <tr>{shuffleWriteQuantiles}</tr> else Nil,
+          if (hasBytesSpilled(stageData)) <tr>{memoryBytesSpilledQuantiles}</tr> else Nil,
+          if (hasBytesSpilled(stageData)) <tr>{diskBytesSpilledQuantiles}</tr> else Nil)
+
+        val quantileHeaders = Seq("Metric", "Min", "25th percentile",
+          "Median", "75th percentile", "Max")
+        // The summary table does not use CSS to stripe rows, which doesn't work with hidden
+        // rows (instead, JavaScript in table.js is used to stripe the non-hidden rows).
+        Some(UIUtils.listingTable(
+          quantileHeaders,
+          identity[Seq[Node]],
+          listings,
+          fixedWidth = true,
+          id = Some("task-summary-table"),
+          stripeRowsWithCss = false))
+      }
 
-      val aggMetrics =
-        <span class="collapse-aggregated-metrics collapse-table"
-              onClick="collapseTable('collapse-aggregated-metrics','aggregated-metrics')">
-          <h4>
-            <span class="collapse-table-arrow arrow-open"></span>
-            <a>Aggregated Metrics by Executor</a>
-          </h4>
-        </span>
-        <div class="aggregated-metrics collapsible-table">
-          {executorTable.toNodeSeq}
-        </div>
+    val executorTable = new ExecutorTable(stageId, stageAttemptId, parent.store)
+
+    val maybeAccumulableTable: Seq[Node] =
+      if (hasAccumulators) { <h4>Accumulators</h4> ++ accumulableTable } else Seq()
+
+    val aggMetrics =
+      <span class="collapse-aggregated-metrics collapse-table"
+            onClick="collapseTable('collapse-aggregated-metrics','aggregated-metrics')">
+        <h4>
+          <span class="collapse-table-arrow arrow-open"></span>
+          <a>Aggregated Metrics by Executor</a>
+        </h4>
+      </span>
+      <div class="aggregated-metrics collapsible-table">
+        {executorTable.toNodeSeq}
+      </div>
 
-      val content =
-        summary ++
-        dagViz ++
-        showAdditionalMetrics ++
-        makeTimeline(
-          // Only show the tasks in the table
-          stageData.taskData.values.toSeq.filter(t => taskIdsInPage.contains(t.taskInfo.taskId)),
-          currentTime) ++
-        <h4>Summary Metrics for <a href="#tasks-section">{numCompleted} Completed Tasks</a></h4> ++
-        <div>{summaryTable.getOrElse("No tasks have reported metrics yet.")}</div> ++
-        aggMetrics ++
-        maybeAccumulableTable ++
-        <h4 id="tasks-section">Tasks ({totalTasksNumStr})</h4> ++
-          taskTableHTML ++ jsForScrollingDownToTaskTable
-      UIUtils.headerSparkPage(stageHeader, content, parent, showVisualization = true)
-    }
+    val content =
+      summary ++
+      dagViz ++
+      showAdditionalMetrics ++
+      makeTimeline(
+        // Only show the tasks in the table
+        taskList.filter { t => taskIdsInPage.contains(t.taskId) },
+        currentTime) ++
+      <h4>Summary Metrics for <a href="#tasks-section">{numCompleted} Completed Tasks</a></h4> ++
+      <div>{summaryTable.getOrElse("No tasks have reported metrics yet.")}</div> ++
+      aggMetrics ++
+      maybeAccumulableTable ++
+      <h4 id="tasks-section">Tasks ({totalTasksNumStr})</h4> ++
+        taskTableHTML ++ jsForScrollingDownToTaskTable
+    UIUtils.headerSparkPage(stageHeader, content, parent, showVisualization = true)
   }
 
-  def makeTimeline(tasks: Seq[TaskUIData], currentTime: Long): Seq[Node] = {
+  def makeTimeline(tasks: Seq[TaskData], currentTime: Long): Seq[Node] = {
     val executorsSet = new HashSet[(String, String)]
     var minLaunchTime = Long.MaxValue
     var maxFinishTime = Long.MinValue
 
     val executorsArrayStr =
-      tasks.sortBy(-_.taskInfo.launchTime).take(MAX_TIMELINE_TASKS).map { taskUIData =>
-        val taskInfo = taskUIData.taskInfo
+      tasks.sortBy(-_.launchTime.getTime()).take(MAX_TIMELINE_TASKS).map { taskInfo =>
         val executorId = taskInfo.executorId
         val host = taskInfo.host
         executorsSet += ((executorId, host))
 
-        val launchTime = taskInfo.launchTime
-        val finishTime = if (!taskInfo.running) taskInfo.finishTime else currentTime
+        val launchTime = taskInfo.launchTime.getTime()
+        val finishTime = taskInfo.duration.map(taskInfo.launchTime.getTime() + _)
+          .getOrElse(currentTime)
         val totalExecutionTime = finishTime - launchTime
         minLaunchTime = launchTime.min(minLaunchTime)
         maxFinishTime = finishTime.max(maxFinishTime)
 
         def toProportion(time: Long) = time.toDouble / totalExecutionTime * 100
 
-        val metricsOpt = taskUIData.metrics
+        val metricsOpt = taskInfo.taskMetrics
         val shuffleReadTime =
           metricsOpt.map(_.shuffleReadMetrics.fetchWaitTime).getOrElse(0L)
         val shuffleReadTimeProportion = toProportion(shuffleReadTime)
@@ -627,14 +602,14 @@ private[ui] class StagePage(parent: StagesTab, store: AppStateStore) extends Web
         val serializationTimeProportion = toProportion(serializationTime)
         val deserializationTime = metricsOpt.map(_.executorDeserializeTime).getOrElse(0L)
         val deserializationTimeProportion = toProportion(deserializationTime)
-        val gettingResultTime = getGettingResultTime(taskUIData.taskInfo, currentTime)
+        val gettingResultTime = getGettingResultTime(taskInfo, currentTime)
         val gettingResultTimeProportion = toProportion(gettingResultTime)
         val schedulerDelay =
           metricsOpt.map(getSchedulerDelay(taskInfo, _, currentTime)).getOrElse(0L)
         val schedulerDelayProportion = toProportion(schedulerDelay)
 
         val executorOverhead = serializationTime + deserializationTime
-        val executorRunTime = if (taskInfo.running) {
+        val executorRunTime = if (taskInfo.duration.isDefined) {
           totalExecutionTime - executorOverhead - gettingResultTime
         } else {
           metricsOpt.map(_.executorRunTime).getOrElse(
@@ -661,7 +636,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStateStore) extends Web
           serializationTimeProportionPos + serializationTimeProportion
 
         val index = taskInfo.index
-        val attempt = taskInfo.attemptNumber
+        val attempt = taskInfo.attempt
 
         val svgTag =
           if (totalExecutionTime == 0) {
@@ -703,7 +678,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStateStore) extends Web
                  |Status: ${taskInfo.status}<br>
                  |Launch Time: ${UIUtils.formatDate(new Date(launchTime))}
                  |${
-                     if (!taskInfo.running) {
+                     if (!taskInfo.duration.isDefined) {
                        s"""<br>Finish Time: ${UIUtils.formatDate(new Date(finishTime))}"""
                      } else {
                         ""
@@ -768,34 +743,40 @@ private[ui] class StagePage(parent: StagesTab, store: AppStateStore) extends Web
 }
 
 private[ui] object StagePage {
-  private[ui] def getGettingResultTime(info: TaskInfo, currentTime: Long): Long = {
-    if (info.gettingResult) {
-      if (info.finished) {
-        info.finishTime - info.gettingResultTime
-      } else {
-        // The task is still fetching the result.
-        currentTime - info.gettingResultTime
-      }
-    } else {
-      0L
+  private[ui] def getGettingResultTime(info: TaskData, currentTime: Long): Long = {
+    info.resultFetchStart match {
+      case Some(start) =>
+        info.duration match {
+          case Some(duration) =>
+            info.launchTime.getTime() + duration - start.getTime()
+
+          case _ =>
+            currentTime - start.getTime()
+        }
+
+      case _ =>
+        0L
     }
   }
 
   private[ui] def getSchedulerDelay(
-      info: TaskInfo, metrics: TaskMetricsUIData, currentTime: Long): Long = {
-    if (info.finished) {
-      val totalExecutionTime = info.finishTime - info.launchTime
-      val executorOverhead = metrics.executorDeserializeTime +
-        metrics.resultSerializationTime
-      math.max(
-        0,
-        totalExecutionTime - metrics.executorRunTime - executorOverhead -
-          getGettingResultTime(info, currentTime))
-    } else {
-      // The task is still running and the metrics like executorRunTime are not available.
-      0L
+      info: TaskData,
+      metrics: TaskMetrics,
+      currentTime: Long): Long = {
+    info.duration match {
+      case Some(duration) =>
+        val executorOverhead = metrics.executorDeserializeTime + metrics.resultSerializationTime
+        math.max(
+          0,
+          duration - metrics.executorRunTime - executorOverhead -
+            getGettingResultTime(info, currentTime))
+
+      case _ =>
+        // The task is still running and the metrics like executorRunTime are not available.
+        0L
     }
   }
+
 }
 
 private[ui] case class TaskTableRowInputData(inputSortable: Long, inputReadable: String)
@@ -824,7 +805,7 @@ private[ui] case class TaskTableRowBytesSpilledData(
 
 /**
  * Contains all data that needs for sorting and generating HTML. Using this one rather than
- * TaskUIData to avoid creating duplicate contents during sorting the data.
+ * TaskData to avoid creating duplicate contents during sorting the data.
  */
 private[ui] class TaskTableRowData(
     val index: Int,
@@ -853,7 +834,7 @@ private[ui] class TaskTableRowData(
     val logs: Map[String, String])
 
 private[ui] class TaskDataSource(
-    tasks: Seq[TaskUIData],
+    tasks: Seq[TaskData],
     hasAccumulators: Boolean,
     hasInput: Boolean,
     hasOutput: Boolean,
@@ -867,7 +848,7 @@ private[ui] class TaskDataSource(
     store: AppStateStore) extends PagedDataSource[TaskTableRowData](pageSize) {
   import StagePage._
 
-  // Convert TaskUIData to TaskTableRowData which contains the final contents to show in the table
+  // Convert TaskData to TaskTableRowData which contains the final contents to show in the table
   // so that we can avoid creating duplicate contents during sorting the data
   private val data = tasks.map(taskRow).sorted(ordering(sortColumn, desc))
 
@@ -883,25 +864,19 @@ private[ui] class TaskDataSource(
 
   def slicedTaskIds: Set[Long] = _slicedTaskIds
 
-  private def taskRow(taskData: TaskUIData): TaskTableRowData = {
-    val info = taskData.taskInfo
-    val metrics = taskData.metrics
-    val duration = taskData.taskDuration.getOrElse(1L)
-    val formatDuration = taskData.taskDuration.map(d => UIUtils.formatDuration(d)).getOrElse("")
+  private def taskRow(info: TaskData): TaskTableRowData = {
+    val metrics = info.taskMetrics
+    val duration = info.duration.getOrElse(1L)
+    val formatDuration = info.duration.map(d => UIUtils.formatDuration(d)).getOrElse("")
     val schedulerDelay = metrics.map(getSchedulerDelay(info, _, currentTime)).getOrElse(0L)
-    val gcTime = metrics.map(_.jvmGCTime).getOrElse(0L)
+    val gcTime = metrics.map(_.jvmGcTime).getOrElse(0L)
     val taskDeserializationTime = metrics.map(_.executorDeserializeTime).getOrElse(0L)
     val serializationTime = metrics.map(_.resultSerializationTime).getOrElse(0L)
     val gettingResultTime = getGettingResultTime(info, currentTime)
 
-    val externalAccumulableReadable = info.accumulables
-      .filterNot(_.internal)
-      .flatMap { a =>
-        (a.name, a.update) match {
-          case (Some(name), Some(update)) => Some(StringEscapeUtils.escapeHtml4(s"$name: $update"))
-          case _ => None
-        }
-      }
+    val externalAccumulableReadable = info.accumulatorUpdates.map { acc =>
+      StringEscapeUtils.escapeHtml4(s"${acc.name}: ${acc.update}")
+    }
     val peakExecutionMemoryUsed = metrics.map(_.peakExecutionMemory).getOrElse(0L)
 
     val maybeInput = metrics.map(_.inputMetrics)
@@ -923,7 +898,7 @@ private[ui] class TaskDataSource(
     val shuffleReadBlockedTimeReadable =
       maybeShuffleRead.map(ms => UIUtils.formatDuration(ms.fetchWaitTime)).getOrElse("")
 
-    val totalShuffleBytes = maybeShuffleRead.map(_.totalBytesRead)
+    val totalShuffleBytes = maybeShuffleRead.map(totalBytesRead)
     val shuffleReadSortable = totalShuffleBytes.getOrElse(0L)
     val shuffleReadReadable = totalShuffleBytes.map(Utils.bytesToString).getOrElse("")
     val shuffleReadRecords = maybeShuffleRead.map(_.recordsRead.toString).getOrElse("")
@@ -1010,12 +985,12 @@ private[ui] class TaskDataSource(
     new TaskTableRowData(
       info.index,
       info.taskId,
-      info.attemptNumber,
+      info.attempt,
       info.speculative,
       info.status,
       info.taskLocality.toString,
       s"${info.executorId} / ${info.host}",
-      info.launchTime,
+      info.launchTime.getTime(),
       duration,
       formatDuration,
       schedulerDelay,
@@ -1030,7 +1005,7 @@ private[ui] class TaskDataSource(
       shuffleRead,
       shuffleWrite,
       bytesSpilled,
-      taskData.errorMessage.getOrElse(""),
+      info.errorMessage.getOrElse(""),
       logs)
   }
 
@@ -1141,7 +1116,7 @@ private[ui] class TaskDataSource(
 private[ui] class TaskPagedTable(
     conf: SparkConf,
     basePath: String,
-    data: Seq[TaskUIData],
+    data: Seq[TaskData],
     hasAccumulators: Boolean,
     hasInput: Boolean,
     hasOutput: Boolean,
@@ -1353,3 +1328,23 @@ private[ui] class TaskPagedTable(
     <td>{errorSummary}{details}</td>
   }
 }
+
+private object ApiHelper {
+
+  def hasInput(stageData: StageData): Boolean = stageData.inputBytes > 0
+
+  def hasOutput(stageData: StageData): Boolean = stageData.outputBytes > 0
+
+  def hasShuffleRead(stageData: StageData): Boolean = stageData.shuffleReadBytes > 0
+
+  def hasShuffleWrite(stageData: StageData): Boolean = stageData.shuffleWriteBytes > 0
+
+  def hasBytesSpilled(stageData: StageData): Boolean = {
+    stageData.diskBytesSpilled > 0 || stageData.memoryBytesSpilled > 0
+  }
+
+  def totalBytesRead(metrics: ShuffleReadMetrics): Long = {
+    metrics.localBytesRead + metrics.remoteBytesRead
+  }
+
+}
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
index 256b726fa7eea..ad623612d3dee 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StageTable.scala
@@ -27,18 +27,21 @@ import scala.xml._
 import org.apache.commons.lang3.StringEscapeUtils
 
 import org.apache.spark.scheduler.StageInfo
+import org.apache.spark.status.AppStateStore
+import org.apache.spark.status.api.v1
+import org.apache.spark.storage.RDDInfo
 import org.apache.spark.ui._
 import org.apache.spark.ui.jobs.UIData.StageUIData
 import org.apache.spark.util.Utils
 
 private[ui] class StageTableBase(
+    store: AppStateStore,
     request: HttpServletRequest,
-    stages: Seq[StageInfo],
+    stages: Seq[v1.StageData],
     tableHeaderID: String,
     stageTag: String,
     basePath: String,
     subPath: String,
-    progressListener: JobProgressListener,
     isFairScheduler: Boolean,
     killEnabled: Boolean,
     isFailedStage: Boolean) {
@@ -77,12 +80,12 @@ private[ui] class StageTableBase(
 
   val toNodeSeq = try {
     new StagePagedTable(
+      store,
       stages,
       tableHeaderID,
       stageTag,
       basePath,
       subPath,
-      progressListener,
       isFairScheduler,
       killEnabled,
       currentTime,
@@ -104,13 +107,13 @@ private[ui] class StageTableBase(
 }
 
 private[ui] class StageTableRowData(
-    val stageInfo: StageInfo,
-    val stageData: Option[StageUIData],
+    val stage: v1.StageData,
+    val option: Option[v1.StageData],
     val stageId: Int,
     val attemptId: Int,
     val schedulingPool: String,
     val descriptionOption: Option[String],
-    val submissionTime: Long,
+    val submissionTime: Date,
     val formattedSubmissionTime: String,
     val duration: Long,
     val formattedDuration: String,
@@ -124,19 +127,20 @@ private[ui] class StageTableRowData(
     val shuffleWriteWithUnit: String)
 
 private[ui] class MissingStageTableRowData(
-    stageInfo: StageInfo,
+    stageInfo: v1.StageData,
     stageId: Int,
     attemptId: Int) extends StageTableRowData(
-  stageInfo, None, stageId, attemptId, "", None, 0, "", -1, "", 0, "", 0, "", 0, "", 0, "")
+  stageInfo, None, stageId, attemptId, "", None, new Date(0), "", -1, "", 0, "", 0, "", 0, "", 0,
+    "")
 
 /** Page showing list of all ongoing and recently finished stages */
 private[ui] class StagePagedTable(
-    stages: Seq[StageInfo],
+    store: AppStateStore,
+    stages: Seq[v1.StageData],
     tableHeaderId: String,
     stageTag: String,
     basePath: String,
     subPath: String,
-    listener: JobProgressListener,
     isFairScheduler: Boolean,
     killEnabled: Boolean,
     currentTime: Long,
@@ -162,8 +166,8 @@ private[ui] class StagePagedTable(
     parameterOtherTable.mkString("&")
 
   override val dataSource = new StageDataSource(
+    store,
     stages,
-    listener,
     currentTime,
     pageSize,
     sortColumn,
@@ -272,10 +276,10 @@ private[ui] class StagePagedTable(
   }
 
   private def rowContent(data: StageTableRowData): Seq[Node] = {
-    data.stageData match {
+    data.option match {
       case None => missingStageRow(data.stageId)
       case Some(stageData) =>
-        val info = data.stageInfo
+        val info = data.stage
 
         {if (data.attemptId > 0) {
           <td>{data.stageId} (retry {data.attemptId})</td>
@@ -299,8 +303,8 @@ private[ui] class StagePagedTable(
         <td>{data.formattedDuration}</td>
         <td class="progress-cell">
           {UIUtils.makeProgressBar(started = stageData.numActiveTasks,
-          completed = stageData.completedIndices.size, failed = stageData.numFailedTasks,
-          skipped = 0, reasonToNumKilled = stageData.reasonToNumKilled, total = info.numTasks)}
+          completed = stageData.numCompleteTasks, failed = stageData.numFailedTasks,
+          skipped = 0, reasonToNumKilled = stageData.killedTasksSummary, total = info.numTasks)}
         </td>
         <td>{data.inputReadWithUnit}</td>
         <td>{data.outputWriteWithUnit}</td>
@@ -316,7 +320,7 @@ private[ui] class StagePagedTable(
     }
   }
 
-  private def failureReasonHtml(s: StageInfo): Seq[Node] = {
+  private def failureReasonHtml(s: v1.StageData): Seq[Node] = {
     val failureReason = s.failureReason.getOrElse("")
     val isMultiline = failureReason.indexOf('\n') >= 0
     // Display the first line by default
@@ -342,7 +346,7 @@ private[ui] class StagePagedTable(
     <td valign="middle">{failureReasonSummary}{details}</td>
   }
 
-  private def makeDescription(s: StageInfo, descriptionOption: Option[String]): Seq[Node] = {
+  private def makeDescription(s: v1.StageData, descriptionOption: Option[String]): Seq[Node] = {
     val basePathUri = UIUtils.prependBaseUri(basePath)
 
     val killLink = if (killEnabled) {
@@ -366,8 +370,8 @@ private[ui] class StagePagedTable(
     val nameLinkUri = s"$basePathUri/stages/stage?id=${s.stageId}&attempt=${s.attemptId}"
     val nameLink = <a href={nameLinkUri} class="name-link">{s.name}</a>
 
-    val cachedRddInfos = s.rddInfos.filter(_.numCachedPartitions > 0)
-    val details = if (s.details.nonEmpty) {
+    val cachedRddInfos = store.rddList().filter { rdd => s.rddIds.contains(rdd.id) }
+    val details = if (s.details != null && s.details.nonEmpty) {
       <span onclick="this.parentNode.querySelector('.stage-details').classList.toggle('collapsed')"
             class="expand-details">
         +details
@@ -402,14 +406,14 @@ private[ui] class StagePagedTable(
 }
 
 private[ui] class StageDataSource(
-    stages: Seq[StageInfo],
-    listener: JobProgressListener,
+    store: AppStateStore,
+    stages: Seq[v1.StageData],
     currentTime: Long,
     pageSize: Int,
     sortColumn: String,
     desc: Boolean) extends PagedDataSource[StageTableRowData](pageSize) {
-  // Convert StageInfo to StageTableRowData which contains the final contents to show in the table
-  // so that we can avoid creating duplicate contents during sorting the data
+  // Convert v1.StageData to StageTableRowData which contains the final contents to show in the
+  // table so that we can avoid creating duplicate contents during sorting the data
   private val data = stages.map(stageRow).sorted(ordering(sortColumn, desc))
 
   private var _slicedStageIds: Set[Int] = _
@@ -422,26 +426,20 @@ private[ui] class StageDataSource(
     r
   }
 
-  private def stageRow(s: StageInfo): StageTableRowData = {
-    val stageDataOption = listener.stageIdToData.get((s.stageId, s.attemptId))
+  private def stageRow(stageData: v1.StageData): StageTableRowData = {
+    val description = stageData.description.getOrElse("")
 
-    if (stageDataOption.isEmpty) {
-      return new MissingStageTableRowData(s, s.stageId, s.attemptId)
-    }
-    val stageData = stageDataOption.get
-
-    val description = stageData.description
-
-    val formattedSubmissionTime = s.submissionTime match {
-      case Some(t) => UIUtils.formatDate(new Date(t))
+    val formattedSubmissionTime = stageData.submissionTime match {
+      case Some(t) => UIUtils.formatDate(t)
       case None => "Unknown"
     }
-    val finishTime = s.completionTime.getOrElse(currentTime)
+    val finishTime = stageData.completionTime.map(_.getTime()).getOrElse(currentTime)
 
     // The submission time for a stage is misleading because it counts the time
     // the stage waits to be launched. (SPARK-10930)
-    val taskLaunchTimes =
-      stageData.taskData.values.map(_.taskInfo.launchTime).filter(_ > 0)
+    val taskLaunchTimes = store
+      .taskList(stageData.stageId, stageData.attemptId, 0, -1, v1.TaskSorting.ID)
+      .map(_.launchTime.getTime()).filter(_ > 0)
     val duration: Option[Long] =
       if (taskLaunchTimes.nonEmpty) {
         val startTime = taskLaunchTimes.min
@@ -459,20 +457,20 @@ private[ui] class StageDataSource(
     val inputReadWithUnit = if (inputRead > 0) Utils.bytesToString(inputRead) else ""
     val outputWrite = stageData.outputBytes
     val outputWriteWithUnit = if (outputWrite > 0) Utils.bytesToString(outputWrite) else ""
-    val shuffleRead = stageData.shuffleReadTotalBytes
+    val shuffleRead = stageData.shuffleReadBytes
     val shuffleReadWithUnit = if (shuffleRead > 0) Utils.bytesToString(shuffleRead) else ""
     val shuffleWrite = stageData.shuffleWriteBytes
     val shuffleWriteWithUnit = if (shuffleWrite > 0) Utils.bytesToString(shuffleWrite) else ""
 
 
     new StageTableRowData(
-      s,
-      stageDataOption,
-      s.stageId,
-      s.attemptId,
+      stageData,
+      Some(stageData),
+      stageData.stageId,
+      stageData.attemptId,
       stageData.schedulingPool,
-      description,
-      s.submissionTime.getOrElse(0),
+      stageData.description,
+      stageData.submissionTime.getOrElse(new Date(0)),
       formattedSubmissionTime,
       duration.getOrElse(-1),
       formattedDuration,
@@ -494,7 +492,7 @@ private[ui] class StageDataSource(
     val ordering: Ordering[StageTableRowData] = sortColumn match {
       case "Stage Id" => Ordering.by(_.stageId)
       case "Pool Name" => Ordering.by(_.schedulingPool)
-      case "Description" => Ordering.by(x => (x.descriptionOption, x.stageInfo.name))
+      case "Description" => Ordering.by(x => (x.descriptionOption, x.stage.name))
       case "Submitted" => Ordering.by(_.submissionTime)
       case "Duration" => Ordering.by(_.duration)
       case "Input" => Ordering.by(_.inputRead)
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala
index 5c8e1aba2ee2f..67c29bdb5f987 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagesTab.scala
@@ -17,38 +17,47 @@
 
 package org.apache.spark.ui.jobs
 
+import java.util.NoSuchElementException
 import javax.servlet.http.HttpServletRequest
 
 import org.apache.spark.scheduler.SchedulingMode
 import org.apache.spark.status.AppStateStore
+import org.apache.spark.status.api.v1.StageStatus
 import org.apache.spark.ui.{SparkUI, SparkUITab}
 
 /** Web UI showing progress status of all stages in the given SparkContext. */
-private[ui] class StagesTab(val parent: SparkUI, store: AppStateStore)
+private[ui] class StagesTab(val parent: SparkUI, val store: AppStateStore)
   extends SparkUITab(parent, "stages") {
 
   val sc = parent.sc
   val conf = parent.conf
   val killEnabled = parent.killEnabled
-  val progressListener = parent.jobProgressListener
-  val operationGraphListener = parent.operationGraphListener
 
   attachPage(new AllStagesPage(this))
   attachPage(new StagePage(this, store))
   attachPage(new PoolPage(this))
 
-  def isFairScheduler: Boolean = progressListener.schedulingMode == Some(SchedulingMode.FAIR)
+  def isFairScheduler: Boolean = {
+    store.environmentInfo().sparkProperties.toMap
+      .get("spark.scheduler.mode")
+      .map { mode => mode == SchedulingMode.FAIR }
+      .getOrElse(false)
+  }
 
   def handleKillRequest(request: HttpServletRequest): Unit = {
     if (killEnabled && parent.securityManager.checkModifyPermissions(request.getRemoteUser)) {
-      val stageId = Option(request.getParameter("id")).map(_.toInt)
-      stageId.foreach { id =>
-        if (progressListener.activeStages.contains(id)) {
-          sc.foreach(_.cancelStage(id, "killed via the Web UI"))
-          // Do a quick pause here to give Spark time to kill the stage so it shows up as
-          // killed after the refresh. Note that this will block the serving thread so the
-          // time should be limited in duration.
-          Thread.sleep(100)
+      Option(request.getParameter("id")).map(_.toInt).foreach { id =>
+        try {
+          val status = store.lastStageAttempt(id).status
+          if (status == StageStatus.ACTIVE || status == StageStatus.PENDING) {
+            sc.foreach(_.cancelStage(id, "killed via the Web UI"))
+            // Do a quick pause here to give Spark time to kill the stage so it shows up as
+            // killed after the refresh. Note that this will block the serving thread so the
+            // time should be limited in duration.
+            Thread.sleep(100)
+          }
+        } catch {
+          case _: NoSuchElementException =>
         }
       }
     }
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
index 43bfe0aacf35b..cf34a245aa471 100644
--- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
+++ b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraph.scala
@@ -35,20 +35,20 @@ import org.apache.spark.storage.{RDDInfo, StorageLevel}
  * nodes and children clusters. Additionally, a graph may also have edges that enter or exit
  * the graph from nodes that belong to adjacent graphs.
  */
-private[ui] case class RDDOperationGraph(
+private[spark] case class RDDOperationGraph(
     edges: Seq[RDDOperationEdge],
     outgoingEdges: Seq[RDDOperationEdge],
     incomingEdges: Seq[RDDOperationEdge],
     rootCluster: RDDOperationCluster)
 
 /** A node in an RDDOperationGraph. This represents an RDD. */
-private[ui] case class RDDOperationNode(id: Int, name: String, cached: Boolean, callsite: String)
+private[spark] case class RDDOperationNode(id: Int, name: String, cached: Boolean, callsite: String)
 
 /**
  * A directed edge connecting two nodes in an RDDOperationGraph.
  * This represents an RDD dependency.
  */
-private[ui] case class RDDOperationEdge(fromId: Int, toId: Int)
+private[spark] case class RDDOperationEdge(fromId: Int, toId: Int)
 
 /**
  * A cluster that groups nodes together in an RDDOperationGraph.
@@ -56,7 +56,7 @@ private[ui] case class RDDOperationEdge(fromId: Int, toId: Int)
  * This represents any grouping of RDDs, including operation scopes (e.g. textFile, flatMap),
  * stages, jobs, or any higher level construct. A cluster may be nested inside of other clusters.
  */
-private[ui] class RDDOperationCluster(val id: String, private var _name: String) {
+private[spark] class RDDOperationCluster(val id: String, private var _name: String) {
   private val _childNodes = new ListBuffer[RDDOperationNode]
   private val _childClusters = new ListBuffer[RDDOperationCluster]
 
@@ -92,7 +92,7 @@ private[ui] class RDDOperationCluster(val id: String, private var _name: String)
   }
 }
 
-private[ui] object RDDOperationGraph extends Logging {
+private[spark] object RDDOperationGraph extends Logging {
 
   val STAGE_CLUSTER_PREFIX = "stage_"
 
diff --git a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraphListener.scala b/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraphListener.scala
deleted file mode 100644
index 37a12a8646938..0000000000000
--- a/core/src/main/scala/org/apache/spark/ui/scope/RDDOperationGraphListener.scala
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ui.scope
-
-import scala.collection.mutable
-
-import org.apache.spark.SparkConf
-import org.apache.spark.scheduler._
-import org.apache.spark.ui.SparkUI
-
-/**
- * A SparkListener that constructs a DAG of RDD operations.
- */
-private[ui] class RDDOperationGraphListener(conf: SparkConf) extends SparkListener {
-
-  // Note: the fate of jobs and stages are tied. This means when we clean up a job,
-  // we always clean up all of its stages. Similarly, when we clean up a stage, we
-  // always clean up its job (and, transitively, other stages in the same job).
-  private[ui] val jobIdToStageIds = new mutable.HashMap[Int, Seq[Int]]
-  private[ui] val jobIdToSkippedStageIds = new mutable.HashMap[Int, Seq[Int]]
-  private[ui] val stageIdToJobId = new mutable.HashMap[Int, Int]
-  private[ui] val stageIdToGraph = new mutable.HashMap[Int, RDDOperationGraph]
-  private[ui] val completedStageIds = new mutable.HashSet[Int]
-
-  // Keep track of the order in which these are inserted so we can remove old ones
-  private[ui] val jobIds = new mutable.ArrayBuffer[Int]
-  private[ui] val stageIds = new mutable.ArrayBuffer[Int]
-
-  // How many root nodes to retain in DAG Graph
-  private[ui] val retainedNodes =
-    conf.getInt("spark.ui.dagGraph.retainedRootRDDs", Int.MaxValue)
-
-  // How many jobs or stages to retain graph metadata for
-  private val retainedJobs =
-    conf.getInt("spark.ui.retainedJobs", SparkUI.DEFAULT_RETAINED_JOBS)
-  private val retainedStages =
-    conf.getInt("spark.ui.retainedStages", SparkUI.DEFAULT_RETAINED_STAGES)
-
-  /**
-   * Return the graph metadata for all stages in the given job.
-   * An empty list is returned if one or more of its stages has been cleaned up.
-   */
-  def getOperationGraphForJob(jobId: Int): Seq[RDDOperationGraph] = synchronized {
-    val skippedStageIds = jobIdToSkippedStageIds.getOrElse(jobId, Seq.empty)
-    val graphs = jobIdToStageIds.getOrElse(jobId, Seq.empty)
-      .flatMap { sid => stageIdToGraph.get(sid) }
-    // Mark any skipped stages as such
-    graphs.foreach { g =>
-      val stageId = g.rootCluster.id.replaceAll(RDDOperationGraph.STAGE_CLUSTER_PREFIX, "").toInt
-      if (skippedStageIds.contains(stageId) && !g.rootCluster.name.contains("skipped")) {
-        g.rootCluster.setName(g.rootCluster.name + " (skipped)")
-      }
-    }
-    graphs
-  }
-
-  /** Return the graph metadata for the given stage, or None if no such information exists. */
-  def getOperationGraphForStage(stageId: Int): Option[RDDOperationGraph] = synchronized {
-    stageIdToGraph.get(stageId)
-  }
-
-  /** On job start, construct a RDDOperationGraph for each stage in the job for display later. */
-  override def onJobStart(jobStart: SparkListenerJobStart): Unit = synchronized {
-    val jobId = jobStart.jobId
-    val stageInfos = jobStart.stageInfos
-
-    jobIds += jobId
-    jobIdToStageIds(jobId) = jobStart.stageInfos.map(_.stageId).sorted
-
-    stageInfos.foreach { stageInfo =>
-      val stageId = stageInfo.stageId
-      stageIds += stageId
-      stageIdToJobId(stageId) = jobId
-      stageIdToGraph(stageId) = RDDOperationGraph.makeOperationGraph(stageInfo, retainedNodes)
-      trimStagesIfNecessary()
-    }
-
-    trimJobsIfNecessary()
-  }
-
-  /** Keep track of stages that have completed. */
-  override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = synchronized {
-    val stageId = stageCompleted.stageInfo.stageId
-    if (stageIdToJobId.contains(stageId)) {
-      // Note: Only do this if the stage has not already been cleaned up
-      // Otherwise, we may never clean this stage from `completedStageIds`
-      completedStageIds += stageCompleted.stageInfo.stageId
-    }
-  }
-
-  /** On job end, find all stages in this job that are skipped and mark them as such. */
-  override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = synchronized {
-    val jobId = jobEnd.jobId
-    jobIdToStageIds.get(jobId).foreach { stageIds =>
-      val skippedStageIds = stageIds.filter { sid => !completedStageIds.contains(sid) }
-      // Note: Only do this if the job has not already been cleaned up
-      // Otherwise, we may never clean this job from `jobIdToSkippedStageIds`
-      jobIdToSkippedStageIds(jobId) = skippedStageIds
-    }
-  }
-
-  /** Clean metadata for old stages if we have exceeded the number to retain. */
-  private def trimStagesIfNecessary(): Unit = {
-    if (stageIds.size >= retainedStages) {
-      val toRemove = math.max(retainedStages / 10, 1)
-      stageIds.take(toRemove).foreach { id => cleanStage(id) }
-      stageIds.trimStart(toRemove)
-    }
-  }
-
-  /** Clean metadata for old jobs if we have exceeded the number to retain. */
-  private def trimJobsIfNecessary(): Unit = {
-    if (jobIds.size >= retainedJobs) {
-      val toRemove = math.max(retainedJobs / 10, 1)
-      jobIds.take(toRemove).foreach { id => cleanJob(id) }
-      jobIds.trimStart(toRemove)
-    }
-  }
-
-  /** Clean metadata for the given stage, its job, and all other stages that belong to the job. */
-  private[ui] def cleanStage(stageId: Int): Unit = {
-    completedStageIds.remove(stageId)
-    stageIdToGraph.remove(stageId)
-    stageIdToJobId.remove(stageId).foreach { jobId => cleanJob(jobId) }
-  }
-
-  /** Clean metadata for the given job and all stages that belong to it. */
-  private[ui] def cleanJob(jobId: Int): Unit = {
-    jobIdToSkippedStageIds.remove(jobId)
-    jobIdToStageIds.remove(jobId).foreach { stageIds =>
-      stageIds.foreach { stageId => cleanStage(stageId) }
-    }
-  }
-
-}
diff --git a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
index 25c4fff77e0ad..dc81d11f15fad 100644
--- a/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/complete_stage_list_json_expectation.json
@@ -2,9 +2,11 @@
   "status" : "COMPLETE",
   "stageId" : 3,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 162,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:07.191GMT",
@@ -23,14 +25,18 @@
   "name" : "count at <console>:17",
   "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line19.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17)\n$line19.$read$$iwC$$iwC$$iwC.<init>(<console>:22)\n$line19.$read$$iwC$$iwC.<init>(<console>:24)\n$line19.$read$$iwC.<init>(<console>:26)\n$line19.$read.<init>(<console>:28)\n$line19.$read$.<init>(<console>:32)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.<init>(<console>:7)\n$line19.$eval$.<clinit>(<console>)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 6, 5 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 }, {
   "status" : "COMPLETE",
   "stageId" : 1,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 3476,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:05.829GMT",
@@ -49,14 +55,18 @@
   "name" : "map at <console>:14",
   "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 1, 0 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 }, {
   "status" : "COMPLETE",
   "stageId" : 0,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 4338,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:04.228GMT",
@@ -75,5 +85,7 @@
   "name" : "count at <console>:15",
   "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:15)\n$line9.$read$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line9.$read$$iwC$$iwC.<init>(<console>:22)\n$line9.$read$$iwC.<init>(<console>:24)\n$line9.$read.<init>(<console>:26)\n$line9.$read$.<init>(<console>:30)\n$line9.$read$.<clinit>(<console>)\n$line9.$eval$.<init>(<console>:7)\n$line9.$eval$.<clinit>(<console>)\n$line9.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 0 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
index b86ba1e65de12..ae787ee6ccaeb 100644
--- a/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/failed_stage_list_json_expectation.json
@@ -2,14 +2,17 @@
   "status" : "FAILED",
   "stageId" : 2,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 7,
   "numFailedTasks" : 1,
+  "numKilledTasks" : 0,
   "executorRunTime" : 278,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:06.296GMT",
   "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT",
   "completionTime" : "2015-02-03T16:43:06.347GMT",
+  "failureReason" : "Job aborted due to stage failure: Task 3 in stage 2.0 failed 1 times, most recent failure: Lost task 3.0 in stage 2.0 (TID 19, localhost): java.lang.RuntimeException: got a 3, failing\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(<console>:18)\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(<console>:17)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1311)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:56)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:196)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:",
   "inputBytes" : 0,
   "inputRecords" : 0,
   "outputBytes" : 0,
@@ -23,5 +26,7 @@
   "name" : "count at <console>:20",
   "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line11.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line11.$read$$iwC$$iwC$$iwC.<init>(<console>:25)\n$line11.$read$$iwC$$iwC.<init>(<console>:27)\n$line11.$read$$iwC.<init>(<console>:29)\n$line11.$read.<init>(<console>:31)\n$line11.$read$.<init>(<console>:35)\n$line11.$read$.<clinit>(<console>)\n$line11.$eval$.<init>(<console>:7)\n$line11.$eval$.<clinit>(<console>)\n$line11.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 3, 2 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
index c108fa61a4318..987ee065a107d 100644
--- a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
@@ -8,8 +8,10 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
index c108fa61a4318..987ee065a107d 100644
--- a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
@@ -8,8 +8,10 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
index 3d7407004d262..063198d3339d0 100644
--- a/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
@@ -8,10 +8,12 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 }, {
   "jobId" : 1,
   "name" : "count at <console>:20",
@@ -22,10 +24,12 @@
   "numCompletedTasks" : 15,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 1,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 1
+  "numFailedStages" : 1,
+  "killedTasksSummary" : { }
 }, {
   "jobId" : 0,
   "name" : "count at <console>:15",
@@ -36,8 +40,10 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
index 10c7e1c0b36fd..d09ab08cdb445 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
@@ -8,8 +8,10 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 }
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index c2f450ba87c6d..6b3be6bcaebeb 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -2,9 +2,11 @@
   "status" : "COMPLETE",
   "stageId" : 1,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 3476,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:05.829GMT",
@@ -23,6 +25,7 @@
   "name" : "map at <console>:14",
   "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
+  "rddIds" : [ 1, 0 ],
   "accumulatorUpdates" : [ ],
   "tasks" : {
     "8" : {
@@ -47,6 +50,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -92,6 +96,7 @@
         "resultSerializationTime" : 0,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -137,6 +142,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -182,6 +188,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -227,6 +234,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -272,6 +280,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -317,6 +326,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -362,6 +372,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -391,12 +402,18 @@
       "taskTime" : 3624,
       "failedTasks" : 0,
       "succeededTasks" : 8,
+      "killedTasks" : 0,
       "inputBytes" : 28000128,
+      "inputRecords" : 0,
       "outputBytes" : 0,
+      "outputRecords" : 0,
       "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
       "shuffleWrite" : 13180,
+      "shuffleWriteRecords" : 0,
       "memoryBytesSpilled" : 0,
       "diskBytesSpilled" : 0
     }
-  }
+  },
+  "killedTasksSummary" : { }
 }
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 506859ae545b1..e5fd260cf4ff8 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -2,9 +2,11 @@
   "status" : "COMPLETE",
   "stageId" : 1,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 3476,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:05.829GMT",
@@ -23,6 +25,7 @@
   "name" : "map at <console>:14",
   "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
+  "rddIds" : [ 1, 0 ],
   "accumulatorUpdates" : [ ],
   "tasks" : {
     "8" : {
@@ -47,6 +50,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -92,6 +96,7 @@
         "resultSerializationTime" : 0,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -137,6 +142,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -182,6 +188,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -227,6 +234,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -272,6 +280,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -317,6 +326,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -362,6 +372,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 3500016,
           "recordsRead" : 0
@@ -391,12 +402,18 @@
       "taskTime" : 3624,
       "failedTasks" : 0,
       "succeededTasks" : 8,
+      "killedTasks" : 0,
       "inputBytes" : 28000128,
+      "inputRecords" : 0,
       "outputBytes" : 0,
+      "outputRecords" : 0,
       "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
       "shuffleWrite" : 13180,
+      "shuffleWriteRecords" : 0,
       "memoryBytesSpilled" : 0,
       "diskBytesSpilled" : 0
     }
-  }
+  },
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
index 6509df1508b30..a3dda890a94a4 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_json_expectation.json
@@ -2,9 +2,11 @@
   "status" : "COMPLETE",
   "stageId" : 3,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 162,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:07.191GMT",
@@ -23,14 +25,49 @@
   "name" : "count at <console>:17",
   "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line19.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17)\n$line19.$read$$iwC$$iwC$$iwC.<init>(<console>:22)\n$line19.$read$$iwC$$iwC.<init>(<console>:24)\n$line19.$read$$iwC.<init>(<console>:26)\n$line19.$read.<init>(<console>:28)\n$line19.$read$.<init>(<console>:32)\n$line19.$read$.<clinit>(<console>)\n$line19.$eval$.<init>(<console>:7)\n$line19.$eval$.<clinit>(<console>)\n$line19.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 6, 5 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
+}, {
+  "status" : "FAILED",
+  "stageId" : 2,
+  "attemptId" : 0,
+  "numTasks" : 8,
+  "numActiveTasks" : 0,
+  "numCompleteTasks" : 7,
+  "numFailedTasks" : 1,
+  "numKilledTasks" : 0,
+  "executorRunTime" : 278,
+  "executorCpuTime" : 0,
+  "submissionTime" : "2015-02-03T16:43:06.296GMT",
+  "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT",
+  "completionTime" : "2015-02-03T16:43:06.347GMT",
+  "failureReason" : "Job aborted due to stage failure: Task 3 in stage 2.0 failed 1 times, most recent failure: Lost task 3.0 in stage 2.0 (TID 19, localhost): java.lang.RuntimeException: got a 3, failing\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(<console>:18)\n\tat $line11.$read$$iwC$$iwC$$iwC$$iwC$$anonfun$1.apply(<console>:17)\n\tat scala.collection.Iterator$$anon$11.next(Iterator.scala:328)\n\tat org.apache.spark.util.Utils$.getIteratorSize(Utils.scala:1311)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.rdd.RDD$$anonfun$count$1.apply(RDD.scala:910)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.SparkContext$$anonfun$runJob$4.apply(SparkContext.scala:1314)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:56)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:196)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:",
+  "inputBytes" : 0,
+  "inputRecords" : 0,
+  "outputBytes" : 0,
+  "outputRecords" : 0,
+  "shuffleReadBytes" : 0,
+  "shuffleReadRecords" : 0,
+  "shuffleWriteBytes" : 0,
+  "shuffleWriteRecords" : 0,
+  "memoryBytesSpilled" : 0,
+  "diskBytesSpilled" : 0,
+  "name" : "count at <console>:20",
+  "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line11.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line11.$read$$iwC$$iwC$$iwC.<init>(<console>:25)\n$line11.$read$$iwC$$iwC.<init>(<console>:27)\n$line11.$read$$iwC.<init>(<console>:29)\n$line11.$read.<init>(<console>:31)\n$line11.$read$.<init>(<console>:35)\n$line11.$read$.<clinit>(<console>)\n$line11.$eval$.<init>(<console>:7)\n$line11.$eval$.<clinit>(<console>)\n$line11.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
+  "schedulingPool" : "default",
+  "rddIds" : [ 3, 2 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 }, {
   "status" : "COMPLETE",
   "stageId" : 1,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 3476,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:05.829GMT",
@@ -49,14 +86,18 @@
   "name" : "map at <console>:14",
   "details" : "org.apache.spark.rdd.RDD.map(RDD.scala:271)\n$line10.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:14)\n$line10.$read$$iwC$$iwC$$iwC.<init>(<console>:19)\n$line10.$read$$iwC$$iwC.<init>(<console>:21)\n$line10.$read$$iwC.<init>(<console>:23)\n$line10.$read.<init>(<console>:25)\n$line10.$read$.<init>(<console>:29)\n$line10.$read$.<clinit>(<console>)\n$line10.$eval$.<init>(<console>:7)\n$line10.$eval$.<clinit>(<console>)\n$line10.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 1, 0 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 }, {
   "status" : "COMPLETE",
   "stageId" : 0,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 4338,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-02-03T16:43:04.228GMT",
@@ -75,31 +116,7 @@
   "name" : "count at <console>:15",
   "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:15)\n$line9.$read$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line9.$read$$iwC$$iwC.<init>(<console>:22)\n$line9.$read$$iwC.<init>(<console>:24)\n$line9.$read.<init>(<console>:26)\n$line9.$read$.<init>(<console>:30)\n$line9.$read$.<clinit>(<console>)\n$line9.$eval$.<init>(<console>:7)\n$line9.$eval$.<clinit>(<console>)\n$line9.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
-}, {
-  "status" : "FAILED",
-  "stageId" : 2,
-  "attemptId" : 0,
-  "numActiveTasks" : 0,
-  "numCompleteTasks" : 7,
-  "numFailedTasks" : 1,
-  "executorRunTime" : 278,
-  "executorCpuTime" : 0,
-  "submissionTime" : "2015-02-03T16:43:06.296GMT",
-  "firstTaskLaunchedTime" : "2015-02-03T16:43:06.296GMT",
-  "completionTime" : "2015-02-03T16:43:06.347GMT",
-  "inputBytes" : 0,
-  "inputRecords" : 0,
-  "outputBytes" : 0,
-  "outputRecords" : 0,
-  "shuffleReadBytes" : 0,
-  "shuffleReadRecords" : 0,
-  "shuffleWriteBytes" : 0,
-  "shuffleWriteRecords" : 0,
-  "memoryBytesSpilled" : 0,
-  "diskBytesSpilled" : 0,
-  "name" : "count at <console>:20",
-  "details" : "org.apache.spark.rdd.RDD.count(RDD.scala:910)\n$line11.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line11.$read$$iwC$$iwC$$iwC.<init>(<console>:25)\n$line11.$read$$iwC$$iwC.<init>(<console>:27)\n$line11.$read$$iwC.<init>(<console>:29)\n$line11.$read.<init>(<console>:31)\n$line11.$read$.<init>(<console>:35)\n$line11.$read$.<clinit>(<console>)\n$line11.$eval$.<init>(<console>:7)\n$line11.$eval$.<clinit>(<console>)\n$line11.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:606)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
-  "schedulingPool" : "default",
-  "accumulatorUpdates" : [ ]
+  "rddIds" : [ 0 ],
+  "accumulatorUpdates" : [ ],
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
index 8496863a93469..ca9700d3ccbb7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_list_with_accumulable_json_expectation.json
@@ -2,9 +2,11 @@
   "status" : "COMPLETE",
   "stageId" : 0,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 120,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-03-16T19:25:36.103GMT",
@@ -23,9 +25,11 @@
   "name" : "foreach at <console>:15",
   "details" : "org.apache.spark.rdd.RDD.foreach(RDD.scala:765)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:15)\n$line9.$read$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line9.$read$$iwC$$iwC.<init>(<console>:22)\n$line9.$read$$iwC.<init>(<console>:24)\n$line9.$read.<init>(<console>:26)\n$line9.$read$.<init>(<console>:30)\n$line9.$read$.<clinit>(<console>)\n$line9.$eval$.<init>(<console>:7)\n$line9.$eval$.<clinit>(<console>)\n$line9.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:483)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
+  "rddIds" : [ 0 ],
   "accumulatorUpdates" : [ {
     "id" : 1,
     "name" : "my counter",
     "value" : "5050"
-  } ]
+  } ],
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index f4cec68fbfdf2..c8057c8435f83 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -20,6 +20,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 49294,
       "recordsRead" : 10000
@@ -64,6 +65,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -108,6 +110,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -152,6 +155,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -196,6 +200,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -240,6 +245,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -284,6 +290,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -328,6 +335,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -372,6 +380,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -416,6 +425,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60489,
       "recordsRead" : 10000
@@ -460,6 +470,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -504,6 +515,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -548,6 +560,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -592,6 +605,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -636,6 +650,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -680,6 +695,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -724,6 +740,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -768,6 +785,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -812,6 +830,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -856,6 +875,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index 496a21c328da9..d156654bd1763 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -25,6 +25,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -74,6 +75,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -123,6 +125,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -172,6 +175,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -221,6 +225,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -270,6 +275,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -319,6 +325,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -368,6 +375,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index 4328dc753c5d4..6cab51b750abd 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -25,6 +25,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -74,6 +75,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -123,6 +125,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -172,6 +175,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -221,6 +225,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -270,6 +275,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -319,6 +325,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
@@ -368,6 +375,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 0,
       "recordsRead" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index 8c571430f3a1f..107cc1defc113 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -20,6 +20,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -64,6 +65,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -108,6 +110,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -152,6 +155,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -196,6 +200,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -240,6 +245,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -284,6 +290,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -328,6 +335,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -372,6 +380,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -416,6 +425,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -460,6 +470,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -504,6 +515,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -548,6 +560,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -592,6 +605,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -636,6 +650,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -680,6 +695,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -724,6 +740,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -768,6 +785,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -812,6 +830,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -856,6 +875,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -900,6 +920,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -944,6 +965,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -988,6 +1010,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1032,6 +1055,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1076,6 +1100,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1120,6 +1145,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1164,6 +1190,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1208,6 +1235,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1252,6 +1280,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1296,6 +1325,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1340,6 +1370,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1384,6 +1415,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1428,6 +1460,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1472,6 +1505,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1516,6 +1550,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1560,6 +1595,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1604,6 +1640,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1648,6 +1685,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1692,6 +1730,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1736,6 +1775,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1780,6 +1820,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1824,6 +1865,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1868,6 +1910,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1912,6 +1955,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -1956,6 +2000,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2000,6 +2045,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2044,6 +2090,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2088,6 +2135,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2132,6 +2180,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -2176,6 +2225,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index 0bd614bdc756e..05c5fc677a035 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -20,6 +20,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -43,10 +44,10 @@
     }
   }
 }, {
-  "taskId" : 1,
-  "index" : 1,
+  "taskId" : 5,
+  "index" : 5,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.502GMT",
+  "launchTime" : "2015-05-06T13:03:06.505GMT",
   "duration" : 350,
   "executorId" : "driver",
   "host" : "localhost",
@@ -55,15 +56,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 31,
+    "executorDeserializeTime" : 30,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 350,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 0,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -82,15 +84,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3934399,
+      "writeTime" : 3675510,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 5,
-  "index" : 5,
+  "taskId" : 1,
+  "index" : 1,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.505GMT",
+  "launchTime" : "2015-05-06T13:03:06.502GMT",
   "duration" : 350,
   "executorId" : "driver",
   "host" : "localhost",
@@ -99,15 +101,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 30,
+    "executorDeserializeTime" : 31,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 350,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -126,15 +129,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3675510,
+      "writeTime" : 3934399,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 0,
-  "index" : 0,
+  "taskId" : 7,
+  "index" : 7,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.494GMT",
+  "launchTime" : "2015-05-06T13:03:06.506GMT",
   "duration" : 349,
   "executorId" : "driver",
   "host" : "localhost",
@@ -143,17 +146,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 32,
+    "executorDeserializeTime" : 31,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 349,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 49294,
+      "bytesRead" : 60488,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -170,13 +174,13 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3842811,
+      "writeTime" : 2579051,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 3,
-  "index" : 3,
+  "taskId" : 4,
+  "index" : 4,
   "attempt" : 0,
   "launchTime" : "2015-05-06T13:03:06.504GMT",
   "duration" : 349,
@@ -193,9 +197,10 @@
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 2,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -214,13 +219,13 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 1311694,
+      "writeTime" : 83022,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 4,
-  "index" : 4,
+  "taskId" : 3,
+  "index" : 3,
   "attempt" : 0,
   "launchTime" : "2015-05-06T13:03:06.504GMT",
   "duration" : 349,
@@ -237,9 +242,10 @@
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -258,15 +264,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 83022,
+      "writeTime" : 1311694,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 7,
-  "index" : 7,
+  "taskId" : 0,
+  "index" : 0,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.506GMT",
+  "launchTime" : "2015-05-06T13:03:06.494GMT",
   "duration" : 349,
   "executorId" : "driver",
   "host" : "localhost",
@@ -275,17 +281,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 31,
+    "executorDeserializeTime" : 32,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 349,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 0,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 60488,
+      "bytesRead" : 49294,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -302,7 +309,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 2579051,
+      "writeTime" : 3842811,
       "recordsWritten" : 10
     }
   }
@@ -328,6 +335,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -372,6 +380,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -416,6 +425,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -460,6 +470,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -504,6 +515,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -527,10 +539,10 @@
     }
   }
 }, {
-  "taskId" : 9,
-  "index" : 9,
+  "taskId" : 19,
+  "index" : 19,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.915GMT",
+  "launchTime" : "2015-05-06T13:03:07.012GMT",
   "duration" : 84,
   "executorId" : "driver",
   "host" : "localhost",
@@ -539,17 +551,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 9,
+    "executorDeserializeTime" : 5,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 84,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 0,
+    "jvmGcTime" : 5,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 60489,
+      "bytesRead" : 70564,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -566,7 +579,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 101664,
+      "writeTime" : 95788,
       "recordsWritten" : 10
     }
   }
@@ -592,6 +605,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -615,10 +629,10 @@
     }
   }
 }, {
-  "taskId" : 19,
-  "index" : 19,
+  "taskId" : 9,
+  "index" : 9,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.012GMT",
+  "launchTime" : "2015-05-06T13:03:06.915GMT",
   "duration" : 84,
   "executorId" : "driver",
   "host" : "localhost",
@@ -627,17 +641,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 5,
+    "executorDeserializeTime" : 9,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 84,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 5,
+    "jvmGcTime" : 0,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 70564,
+      "bytesRead" : 60489,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -654,15 +669,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95788,
+      "writeTime" : 101664,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 14,
-  "index" : 14,
+  "taskId" : 20,
+  "index" : 20,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.925GMT",
+  "launchTime" : "2015-05-06T13:03:07.014GMT",
   "duration" : 83,
   "executorId" : "driver",
   "host" : "localhost",
@@ -671,15 +686,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 6,
+    "executorDeserializeTime" : 3,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 83,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 0,
+    "jvmGcTime" : 5,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -698,15 +714,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95646,
+      "writeTime" : 97716,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 20,
-  "index" : 20,
+  "taskId" : 14,
+  "index" : 14,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.014GMT",
+  "launchTime" : "2015-05-06T13:03:06.925GMT",
   "duration" : 83,
   "executorId" : "driver",
   "host" : "localhost",
@@ -715,15 +731,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 3,
+    "executorDeserializeTime" : 6,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 83,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 5,
+    "jvmGcTime" : 0,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -742,7 +759,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 97716,
+      "writeTime" : 95646,
       "recordsWritten" : 10
     }
   }
@@ -768,6 +785,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -812,6 +830,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -835,10 +854,10 @@
     }
   }
 }, {
-  "taskId" : 13,
-  "index" : 13,
+  "taskId" : 15,
+  "index" : 15,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.924GMT",
+  "launchTime" : "2015-05-06T13:03:06.928GMT",
   "duration" : 76,
   "executorId" : "driver",
   "host" : "localhost",
@@ -847,7 +866,7 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 9,
+    "executorDeserializeTime" : 3,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 76,
     "executorCpuTime" : 0,
@@ -856,6 +875,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -874,7 +894,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95004,
+      "writeTime" : 602780,
       "recordsWritten" : 10
     }
   }
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 0bd614bdc756e..05c5fc677a035 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -20,6 +20,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -43,10 +44,10 @@
     }
   }
 }, {
-  "taskId" : 1,
-  "index" : 1,
+  "taskId" : 5,
+  "index" : 5,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.502GMT",
+  "launchTime" : "2015-05-06T13:03:06.505GMT",
   "duration" : 350,
   "executorId" : "driver",
   "host" : "localhost",
@@ -55,15 +56,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 31,
+    "executorDeserializeTime" : 30,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 350,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 0,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -82,15 +84,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3934399,
+      "writeTime" : 3675510,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 5,
-  "index" : 5,
+  "taskId" : 1,
+  "index" : 1,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.505GMT",
+  "launchTime" : "2015-05-06T13:03:06.502GMT",
   "duration" : 350,
   "executorId" : "driver",
   "host" : "localhost",
@@ -99,15 +101,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 30,
+    "executorDeserializeTime" : 31,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 350,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -126,15 +129,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3675510,
+      "writeTime" : 3934399,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 0,
-  "index" : 0,
+  "taskId" : 7,
+  "index" : 7,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.494GMT",
+  "launchTime" : "2015-05-06T13:03:06.506GMT",
   "duration" : 349,
   "executorId" : "driver",
   "host" : "localhost",
@@ -143,17 +146,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 32,
+    "executorDeserializeTime" : 31,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 349,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 49294,
+      "bytesRead" : 60488,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -170,13 +174,13 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 3842811,
+      "writeTime" : 2579051,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 3,
-  "index" : 3,
+  "taskId" : 4,
+  "index" : 4,
   "attempt" : 0,
   "launchTime" : "2015-05-06T13:03:06.504GMT",
   "duration" : 349,
@@ -193,9 +197,10 @@
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 2,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -214,13 +219,13 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 1311694,
+      "writeTime" : 83022,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 4,
-  "index" : 4,
+  "taskId" : 3,
+  "index" : 3,
   "attempt" : 0,
   "launchTime" : "2015-05-06T13:03:06.504GMT",
   "duration" : 349,
@@ -237,9 +242,10 @@
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 1,
+    "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -258,15 +264,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 83022,
+      "writeTime" : 1311694,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 7,
-  "index" : 7,
+  "taskId" : 0,
+  "index" : 0,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.506GMT",
+  "launchTime" : "2015-05-06T13:03:06.494GMT",
   "duration" : 349,
   "executorId" : "driver",
   "host" : "localhost",
@@ -275,17 +281,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 31,
+    "executorDeserializeTime" : 32,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 349,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
     "jvmGcTime" : 7,
-    "resultSerializationTime" : 0,
+    "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 60488,
+      "bytesRead" : 49294,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -302,7 +309,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 2579051,
+      "writeTime" : 3842811,
       "recordsWritten" : 10
     }
   }
@@ -328,6 +335,7 @@
     "resultSerializationTime" : 2,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -372,6 +380,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -416,6 +425,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -460,6 +470,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -504,6 +515,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -527,10 +539,10 @@
     }
   }
 }, {
-  "taskId" : 9,
-  "index" : 9,
+  "taskId" : 19,
+  "index" : 19,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.915GMT",
+  "launchTime" : "2015-05-06T13:03:07.012GMT",
   "duration" : 84,
   "executorId" : "driver",
   "host" : "localhost",
@@ -539,17 +551,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 9,
+    "executorDeserializeTime" : 5,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 84,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 0,
+    "jvmGcTime" : 5,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 60489,
+      "bytesRead" : 70564,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -566,7 +579,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 101664,
+      "writeTime" : 95788,
       "recordsWritten" : 10
     }
   }
@@ -592,6 +605,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -615,10 +629,10 @@
     }
   }
 }, {
-  "taskId" : 19,
-  "index" : 19,
+  "taskId" : 9,
+  "index" : 9,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.012GMT",
+  "launchTime" : "2015-05-06T13:03:06.915GMT",
   "duration" : 84,
   "executorId" : "driver",
   "host" : "localhost",
@@ -627,17 +641,18 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 5,
+    "executorDeserializeTime" : 9,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 84,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 5,
+    "jvmGcTime" : 0,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
-      "bytesRead" : 70564,
+      "bytesRead" : 60489,
       "recordsRead" : 10000
     },
     "outputMetrics" : {
@@ -654,15 +669,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95788,
+      "writeTime" : 101664,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 14,
-  "index" : 14,
+  "taskId" : 20,
+  "index" : 20,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.925GMT",
+  "launchTime" : "2015-05-06T13:03:07.014GMT",
   "duration" : 83,
   "executorId" : "driver",
   "host" : "localhost",
@@ -671,15 +686,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 6,
+    "executorDeserializeTime" : 3,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 83,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 0,
+    "jvmGcTime" : 5,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -698,15 +714,15 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95646,
+      "writeTime" : 97716,
       "recordsWritten" : 10
     }
   }
 }, {
-  "taskId" : 20,
-  "index" : 20,
+  "taskId" : 14,
+  "index" : 14,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:07.014GMT",
+  "launchTime" : "2015-05-06T13:03:06.925GMT",
   "duration" : 83,
   "executorId" : "driver",
   "host" : "localhost",
@@ -715,15 +731,16 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 3,
+    "executorDeserializeTime" : 6,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 83,
     "executorCpuTime" : 0,
     "resultSize" : 2010,
-    "jvmGcTime" : 5,
+    "jvmGcTime" : 0,
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -742,7 +759,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 97716,
+      "writeTime" : 95646,
       "recordsWritten" : 10
     }
   }
@@ -768,6 +785,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 60488,
       "recordsRead" : 10000
@@ -812,6 +830,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -835,10 +854,10 @@
     }
   }
 }, {
-  "taskId" : 13,
-  "index" : 13,
+  "taskId" : 15,
+  "index" : 15,
   "attempt" : 0,
-  "launchTime" : "2015-05-06T13:03:06.924GMT",
+  "launchTime" : "2015-05-06T13:03:06.928GMT",
   "duration" : 76,
   "executorId" : "driver",
   "host" : "localhost",
@@ -847,7 +866,7 @@
   "speculative" : false,
   "accumulatorUpdates" : [ ],
   "taskMetrics" : {
-    "executorDeserializeTime" : 9,
+    "executorDeserializeTime" : 3,
     "executorDeserializeCpuTime" : 0,
     "executorRunTime" : 76,
     "executorCpuTime" : 0,
@@ -856,6 +875,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -874,7 +894,7 @@
     },
     "shuffleWriteMetrics" : {
       "bytesWritten" : 1710,
-      "writeTime" : 95004,
+      "writeTime" : 602780,
       "recordsWritten" : 10
     }
   }
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index b58f1a51ba481..7d670dbc1f6be 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -20,6 +20,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -64,6 +65,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -108,6 +110,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -152,6 +155,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -196,6 +200,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -240,6 +245,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -284,6 +290,7 @@
     "resultSerializationTime" : 1,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -328,6 +335,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -372,6 +380,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -416,6 +425,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -460,6 +470,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -504,6 +515,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -548,6 +560,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -592,6 +605,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -636,6 +650,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -680,6 +695,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70565,
       "recordsRead" : 10000
@@ -724,6 +740,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -768,6 +785,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -812,6 +830,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
@@ -856,6 +875,7 @@
     "resultSerializationTime" : 0,
     "memoryBytesSpilled" : 0,
     "diskBytesSpilled" : 0,
+    "peakExecutionMemory" : 0,
     "inputMetrics" : {
       "bytesRead" : 70564,
       "recordsRead" : 10000
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index a449926ee7dc6..eab3b2d4a4155 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -2,9 +2,11 @@
   "status" : "COMPLETE",
   "stageId" : 0,
   "attemptId" : 0,
+  "numTasks" : 8,
   "numActiveTasks" : 0,
   "numCompleteTasks" : 8,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "executorRunTime" : 120,
   "executorCpuTime" : 0,
   "submissionTime" : "2015-03-16T19:25:36.103GMT",
@@ -23,6 +25,7 @@
   "name" : "foreach at <console>:15",
   "details" : "org.apache.spark.rdd.RDD.foreach(RDD.scala:765)\n$line9.$read$$iwC$$iwC$$iwC$$iwC.<init>(<console>:15)\n$line9.$read$$iwC$$iwC$$iwC.<init>(<console>:20)\n$line9.$read$$iwC$$iwC.<init>(<console>:22)\n$line9.$read$$iwC.<init>(<console>:24)\n$line9.$read.<init>(<console>:26)\n$line9.$read$.<init>(<console>:30)\n$line9.$read$.<clinit>(<console>)\n$line9.$eval$.<init>(<console>:7)\n$line9.$eval$.<clinit>(<console>)\n$line9.$eval.$print(<console>)\nsun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\nsun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\nsun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\njava.lang.reflect.Method.invoke(Method.java:483)\norg.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:852)\norg.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1125)\norg.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:674)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:705)\norg.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:669)",
   "schedulingPool" : "default",
+  "rddIds" : [ 0 ],
   "accumulatorUpdates" : [ {
     "id" : 1,
     "name" : "my counter",
@@ -56,6 +59,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -106,6 +110,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -156,6 +161,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -206,6 +212,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -256,6 +263,7 @@
         "resultSerializationTime" : 1,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -306,6 +314,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -356,6 +365,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -406,6 +416,7 @@
         "resultSerializationTime" : 2,
         "memoryBytesSpilled" : 0,
         "diskBytesSpilled" : 0,
+        "peakExecutionMemory" : 0,
         "inputMetrics" : {
           "bytesRead" : 0,
           "recordsRead" : 0
@@ -435,12 +446,18 @@
       "taskTime" : 418,
       "failedTasks" : 0,
       "succeededTasks" : 8,
+      "killedTasks" : 0,
       "inputBytes" : 0,
+      "inputRecords" : 0,
       "outputBytes" : 0,
+      "outputRecords" : 0,
       "shuffleRead" : 0,
+      "shuffleReadRecords" : 0,
       "shuffleWrite" : 0,
+      "shuffleWriteRecords" : 0,
       "memoryBytesSpilled" : 0,
       "diskBytesSpilled" : 0
     }
-  }
+  },
+  "killedTasksSummary" : { }
 }
diff --git a/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
index 3d7407004d262..063198d3339d0 100644
--- a/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
@@ -8,10 +8,12 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 }, {
   "jobId" : 1,
   "name" : "count at <console>:20",
@@ -22,10 +24,12 @@
   "numCompletedTasks" : 15,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 1,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 1
+  "numFailedStages" : 1,
+  "killedTasksSummary" : { }
 }, {
   "jobId" : 0,
   "name" : "count at <console>:15",
@@ -36,8 +40,10 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
index 6a9bafd6b2191..e032de41b8e7b 100644
--- a/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
@@ -8,10 +8,12 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 }, {
   "jobId" : 0,
   "name" : "count at <console>:15",
@@ -22,8 +24,10 @@
   "numCompletedTasks" : 8,
   "numSkippedTasks" : 0,
   "numFailedTasks" : 0,
+  "numKilledTasks" : 0,
   "numActiveStages" : 0,
   "numCompletedStages" : 1,
   "numSkippedStages" : 0,
-  "numFailedStages" : 0
+  "numFailedStages" : 0,
+  "killedTasksSummary" : { }
 } ]
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index d2a318a680baa..86606f2813225 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -23,6 +23,7 @@ import java.util.zip.ZipInputStream
 import javax.servlet._
 import javax.servlet.http.{HttpServletRequest, HttpServletRequestWrapper, HttpServletResponse}
 
+import scala.collection.JavaConverters._
 import scala.concurrent.duration._
 import scala.language.postfixOps
 
@@ -44,8 +45,8 @@ import org.scalatest.selenium.WebBrowser
 
 import org.apache.spark._
 import org.apache.spark.deploy.history.config._
+import org.apache.spark.status.api.v1.JobData
 import org.apache.spark.ui.SparkUI
-import org.apache.spark.ui.jobs.UIData.JobUIData
 import org.apache.spark.util.{ResetSystemProperties, Utils}
 
 /**
@@ -260,7 +261,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
 
     val badStageAttemptId = getContentAndCode("applications/local-1422981780767/stages/1/1")
     badStageAttemptId._1 should be (HttpServletResponse.SC_NOT_FOUND)
-    badStageAttemptId._3 should be (Some("unknown attempt for stage 1.  Found attempts: [0]"))
+    badStageAttemptId._3 should be (Some("unknown attempt 1 for stage 1."))
 
     val badStageId2 = getContentAndCode("applications/local-1422981780767/stages/flimflam")
     badStageId2._1 should be (HttpServletResponse.SC_NOT_FOUND)
@@ -494,12 +495,12 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
       }
     }
 
-    def completedJobs(): Seq[JobUIData] = {
-      getAppUI.jobProgressListener.completedJobs
+    def completedJobs(): Seq[JobData] = {
+      getAppUI.store.jobsList(List(JobExecutionStatus.SUCCEEDED).asJava)
     }
 
-    def activeJobs(): Seq[JobUIData] = {
-      getAppUI.jobProgressListener.activeJobs.values.toSeq
+    def activeJobs(): Seq[JobData] = {
+      getAppUI.store.jobsList(List(JobExecutionStatus.RUNNING).asJava)
     }
 
     activeJobs() should have size 0
diff --git a/core/src/test/scala/org/apache/spark/status/AppStateListenerSuite.scala b/core/src/test/scala/org/apache/spark/status/AppStateListenerSuite.scala
index d47d52f2b4e34..b1c30a9695ddc 100644
--- a/core/src/test/scala/org/apache/spark/status/AppStateListenerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/status/AppStateListenerSuite.scala
@@ -151,7 +151,7 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
     check[JobDataWrapper](1) { job =>
       assert(job.info.jobId === 1)
       assert(job.info.name === stages.last.name)
-      assert(job.info.description === Some(stages.last.details))
+      assert(job.info.description === None)
       assert(job.info.status === JobExecutionStatus.RUNNING)
       assert(job.info.submissionTime === Some(new Date(time)))
     }
@@ -175,6 +175,7 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
     check[StageDataWrapper](key(stages.head)) { stage =>
       assert(stage.info.status === v1.StageStatus.ACTIVE)
       assert(stage.info.submissionTime === Some(new Date(stages.head.submissionTime.get)))
+      assert(stage.info.numTasks === stages.head.numTasks)
     }
 
     // Start tasks from stage 1
@@ -265,12 +266,7 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
       "taskType", TaskResultLost, s1Tasks.head, null))
 
     time += 1
-    val reattempt = {
-      val orig = s1Tasks.head
-      // Task reattempts have a different ID, but the same index as the original.
-      new TaskInfo(nextTaskId(), orig.index, orig.attemptNumber + 1, time, orig.executorId,
-        s"${orig.executorId}.example.com", TaskLocality.PROCESS_LOCAL, orig.speculative)
-    }
+    val reattempt = newAttempt(s1Tasks.head, nextTaskId())
     listener.onTaskStart(SparkListenerTaskStart(stages.head.stageId, stages.head.attemptId,
       reattempt))
 
@@ -297,8 +293,33 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
       assert(task.info.attempt === reattempt.attemptNumber)
     }
 
+    // Kill one task, restart it.
+    time += 1
+    val killed = s1Tasks.drop(1).head
+    killed.finishTime = time
+    killed.failed = true
+    listener.onTaskEnd(SparkListenerTaskEnd(stages.head.stageId, stages.head.attemptId,
+      "taskType", TaskKilled("killed"), killed, null))
+
+    check[JobDataWrapper](1) { job =>
+      assert(job.info.numKilledTasks === 1)
+    }
+
+    check[StageDataWrapper](key(stages.head)) { stage =>
+      assert(stage.info.numKilledTasks === 1)
+    }
+
+    check[TaskDataWrapper](killed.taskId) { task =>
+      assert(task.info.index === killed.index)
+      assert(task.info.errorMessage === Some("killed"))
+    }
+
+    val reattempt2 = newAttempt(killed, nextTaskId())
+    listener.onTaskStart(SparkListenerTaskStart(stages.head.stageId, stages.head.attemptId,
+      reattempt2))
+
     // Succeed all tasks in stage 1.
-    val pending = s1Tasks.drop(1) ++ Seq(reattempt)
+    val pending = s1Tasks.drop(2) ++ Seq(reattempt, reattempt2)
 
     val s1Metrics = TaskMetrics.empty
     s1Metrics.setExecutorCpuTime(2L)
@@ -313,12 +334,14 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
 
     check[JobDataWrapper](1) { job =>
       assert(job.info.numFailedTasks === 1)
+      assert(job.info.numKilledTasks === 1)
       assert(job.info.numActiveTasks === 0)
       assert(job.info.numCompletedTasks === pending.size)
     }
 
     check[StageDataWrapper](key(stages.head)) { stage =>
       assert(stage.info.numFailedTasks === 1)
+      assert(stage.info.numKilledTasks === 1)
       assert(stage.info.numActiveTasks === 0)
       assert(stage.info.numCompleteTasks === pending.size)
     }
@@ -333,7 +356,7 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
       }
     }
 
-    assert(store.count(classOf[TaskDataWrapper]) === pending.size + 1)
+    assert(store.count(classOf[TaskDataWrapper]) === pending.size + 2)
 
     // End stage 1.
     time += 1
@@ -407,6 +430,7 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
       assert(stage.info.numFailedTasks === s2Tasks.size)
       assert(stage.info.numActiveTasks === 0)
       assert(stage.info.numCompleteTasks === 0)
+      assert(stage.info.failureReason === stages.last.failureReason)
     }
 
     // - Re-submit stage 2, all tasks, and succeed them and the stage.
@@ -772,4 +796,10 @@ class AppStateListenerSuite extends SparkFunSuite with BeforeAndAfter {
     fn(value)
   }
 
+  private def newAttempt(orig: TaskInfo, nextId: Long): TaskInfo = {
+    // Task reattempts have a different ID, but the same index as the original.
+    new TaskInfo(nextId, orig.index, orig.attemptNumber + 1, time, orig.executorId,
+      s"${orig.executorId}.example.com", TaskLocality.PROCESS_LOCAL, orig.speculative)
+  }
+
 }
diff --git a/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala b/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala
deleted file mode 100644
index 1bfb0c1547ec4..0000000000000
--- a/core/src/test/scala/org/apache/spark/status/api/v1/AllStagesResourceSuite.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.status.api.v1
-
-import java.util.Date
-
-import scala.collection.mutable.LinkedHashMap
-
-import org.apache.spark.SparkFunSuite
-import org.apache.spark.scheduler.{StageInfo, TaskInfo, TaskLocality}
-import org.apache.spark.ui.jobs.UIData.{StageUIData, TaskUIData}
-
-class AllStagesResourceSuite extends SparkFunSuite {
-
-  def getFirstTaskLaunchTime(taskLaunchTimes: Seq[Long]): Option[Date] = {
-    val tasks = new LinkedHashMap[Long, TaskUIData]
-    taskLaunchTimes.zipWithIndex.foreach { case (time, idx) =>
-      tasks(idx.toLong) = TaskUIData(
-        new TaskInfo(idx, idx, 1, time, "", "", TaskLocality.ANY, false), None)
-    }
-
-    val stageUiData = new StageUIData()
-    stageUiData.taskData = tasks
-    val status = StageStatus.ACTIVE
-    val stageInfo = new StageInfo(
-      1, 1, "stage 1", 10, Seq.empty, Seq.empty, "details abc")
-    val stageData = AllStagesResource.stageUiToStageData(status, stageInfo, stageUiData, false)
-
-    stageData.firstTaskLaunchedTime
-  }
-
-  test("firstTaskLaunchedTime when there are no tasks") {
-    val result = getFirstTaskLaunchTime(Seq())
-    assert(result == None)
-  }
-
-  test("firstTaskLaunchedTime when there are tasks but none launched") {
-    val result = getFirstTaskLaunchTime(Seq(-100L, -200L, -300L))
-    assert(result == None)
-  }
-
-  test("firstTaskLaunchedTime when there are tasks and some launched") {
-    val result = getFirstTaskLaunchTime(Seq(-100L, 1449255596000L, 1449255597000L))
-    assert(result == Some(new Date(1449255596000L)))
-  }
-
-}
diff --git a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
index d39660276442f..86ff0817acbd3 100644
--- a/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/StagePageSuite.scala
@@ -29,8 +29,8 @@ import org.apache.spark._
 import org.apache.spark.executor.TaskMetrics
 import org.apache.spark.scheduler._
 import org.apache.spark.status.AppStateStore
-import org.apache.spark.ui.jobs.{JobProgressListener, StagePage, StagesTab}
-import org.apache.spark.ui.scope.RDDOperationGraphListener
+import org.apache.spark.ui.jobs.{StagePage, StagesTab}
+import org.apache.spark.util.Utils
 
 class StagePageSuite extends SparkFunSuite with LocalSparkContext {
 
@@ -55,38 +55,40 @@ class StagePageSuite extends SparkFunSuite with LocalSparkContext {
    * This also runs a dummy stage to populate the page with useful content.
    */
   private def renderStagePage(conf: SparkConf): Seq[Node] = {
-    val store = mock(classOf[AppStateStore])
-    when(store.executorSummary(anyString())).thenReturn(None)
+    val bus = new ReplayListenerBus()
+    val store = AppStateStore.createTempStore(conf, bus)
 
-    val jobListener = new JobProgressListener(conf)
-    val graphListener = new RDDOperationGraphListener(conf)
-    val tab = mock(classOf[StagesTab], RETURNS_SMART_NULLS)
-    val request = mock(classOf[HttpServletRequest])
-    when(tab.conf).thenReturn(conf)
-    when(tab.progressListener).thenReturn(jobListener)
-    when(tab.operationGraphListener).thenReturn(graphListener)
-    when(tab.appName).thenReturn("testing")
-    when(tab.headerTabs).thenReturn(Seq.empty)
-    when(request.getParameter("id")).thenReturn("0")
-    when(request.getParameter("attempt")).thenReturn("0")
-    val page = new StagePage(tab, store)
+    try {
+      val tab = mock(classOf[StagesTab], RETURNS_SMART_NULLS)
+      when(tab.store).thenReturn(store)
 
-    // Simulate a stage in job progress listener
-    val stageInfo = new StageInfo(0, 0, "dummy", 1, Seq.empty, Seq.empty, "details")
-    // Simulate two tasks to test PEAK_EXECUTION_MEMORY correctness
-    (1 to 2).foreach {
-      taskId =>
-        val taskInfo = new TaskInfo(taskId, taskId, 0, 0, "0", "localhost", TaskLocality.ANY, false)
-        jobListener.onStageSubmitted(SparkListenerStageSubmitted(stageInfo))
-        jobListener.onTaskStart(SparkListenerTaskStart(0, 0, taskInfo))
-        taskInfo.markFinished(TaskState.FINISHED, System.currentTimeMillis())
-        val taskMetrics = TaskMetrics.empty
-        taskMetrics.incPeakExecutionMemory(peakExecutionMemory)
-        jobListener.onTaskEnd(
-          SparkListenerTaskEnd(0, 0, "result", Success, taskInfo, taskMetrics))
+      val request = mock(classOf[HttpServletRequest])
+      when(tab.conf).thenReturn(conf)
+      when(tab.appName).thenReturn("testing")
+      when(tab.headerTabs).thenReturn(Seq.empty)
+      when(request.getParameter("id")).thenReturn("0")
+      when(request.getParameter("attempt")).thenReturn("0")
+      val page = new StagePage(tab, store)
+
+      // Simulate a stage in job progress listener
+      val stageInfo = new StageInfo(0, 0, "dummy", 1, Seq.empty, Seq.empty, "details")
+      // Simulate two tasks to test PEAK_EXECUTION_MEMORY correctness
+      (1 to 2).foreach {
+        taskId =>
+          val taskInfo = new TaskInfo(taskId, taskId, 0, 0, "0", "localhost", TaskLocality.ANY,
+            false)
+          bus.postToAll(SparkListenerStageSubmitted(stageInfo))
+          bus.postToAll(SparkListenerTaskStart(0, 0, taskInfo))
+          taskInfo.markFinished(TaskState.FINISHED, System.currentTimeMillis())
+          val taskMetrics = TaskMetrics.empty
+          taskMetrics.incPeakExecutionMemory(peakExecutionMemory)
+          bus.postToAll(SparkListenerTaskEnd(0, 0, "result", Success, taskInfo, taskMetrics))
+      }
+      bus.postToAll(SparkListenerStageCompleted(stageInfo))
+      page.render(request)
+    } finally {
+      store.close()
     }
-    jobListener.onStageCompleted(SparkListenerStageCompleted(stageInfo))
-    page.render(request)
   }
 
 }
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index bdd148875e38a..bbc888cb5906a 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -529,7 +529,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
     }
   }
 
-  test("stage & job retention") {
+  ignore("stage & job retention") {
     val conf = new SparkConf()
       .setMaster("local")
       .setAppName("test")
@@ -675,34 +675,36 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser with Matchers with B
         sc.parallelize(Seq(1, 2, 3)).map(identity).groupBy(identity).map(identity).groupBy(identity)
       rdd.count()
 
-      val stage0 = Source.fromURL(sc.ui.get.webUrl +
-        "/stages/stage/?id=0&attempt=0&expandDagViz=true").mkString
-      assert(stage0.contains("digraph G {\n  subgraph clusterstage_0 {\n    " +
-        "label=&quot;Stage 0&quot;;\n    subgraph "))
-      assert(stage0.contains("{\n      label=&quot;parallelize&quot;;\n      " +
-        "0 [label=&quot;ParallelCollectionRDD [0]"))
-      assert(stage0.contains("{\n      label=&quot;map&quot;;\n      " +
-        "1 [label=&quot;MapPartitionsRDD [1]"))
-      assert(stage0.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-        "2 [label=&quot;MapPartitionsRDD [2]"))
-
-      val stage1 = Source.fromURL(sc.ui.get.webUrl +
-        "/stages/stage/?id=1&attempt=0&expandDagViz=true").mkString
-      assert(stage1.contains("digraph G {\n  subgraph clusterstage_1 {\n    " +
-        "label=&quot;Stage 1&quot;;\n    subgraph "))
-      assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-        "3 [label=&quot;ShuffledRDD [3]"))
-      assert(stage1.contains("{\n      label=&quot;map&quot;;\n      " +
-        "4 [label=&quot;MapPartitionsRDD [4]"))
-      assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-        "5 [label=&quot;MapPartitionsRDD [5]"))
-
-      val stage2 = Source.fromURL(sc.ui.get.webUrl +
-        "/stages/stage/?id=2&attempt=0&expandDagViz=true").mkString
-      assert(stage2.contains("digraph G {\n  subgraph clusterstage_2 {\n    " +
-        "label=&quot;Stage 2&quot;;\n    subgraph "))
-      assert(stage2.contains("{\n      label=&quot;groupBy&quot;;\n      " +
-        "6 [label=&quot;ShuffledRDD [6]"))
+      eventually(timeout(5 seconds), interval(100 milliseconds)) {
+        val stage0 = Source.fromURL(sc.ui.get.webUrl +
+          "/stages/stage/?id=0&attempt=0&expandDagViz=true").mkString
+        assert(stage0.contains("digraph G {\n  subgraph clusterstage_0 {\n    " +
+          "label=&quot;Stage 0&quot;;\n    subgraph "))
+        assert(stage0.contains("{\n      label=&quot;parallelize&quot;;\n      " +
+          "0 [label=&quot;ParallelCollectionRDD [0]"))
+        assert(stage0.contains("{\n      label=&quot;map&quot;;\n      " +
+          "1 [label=&quot;MapPartitionsRDD [1]"))
+        assert(stage0.contains("{\n      label=&quot;groupBy&quot;;\n      " +
+          "2 [label=&quot;MapPartitionsRDD [2]"))
+
+        val stage1 = Source.fromURL(sc.ui.get.webUrl +
+          "/stages/stage/?id=1&attempt=0&expandDagViz=true").mkString
+        assert(stage1.contains("digraph G {\n  subgraph clusterstage_1 {\n    " +
+          "label=&quot;Stage 1&quot;;\n    subgraph "))
+        assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
+          "3 [label=&quot;ShuffledRDD [3]"))
+        assert(stage1.contains("{\n      label=&quot;map&quot;;\n      " +
+          "4 [label=&quot;MapPartitionsRDD [4]"))
+        assert(stage1.contains("{\n      label=&quot;groupBy&quot;;\n      " +
+          "5 [label=&quot;MapPartitionsRDD [5]"))
+
+        val stage2 = Source.fromURL(sc.ui.get.webUrl +
+          "/stages/stage/?id=2&attempt=0&expandDagViz=true").mkString
+        assert(stage2.contains("digraph G {\n  subgraph clusterstage_2 {\n    " +
+          "label=&quot;Stage 2&quot;;\n    subgraph "))
+        assert(stage2.contains("{\n      label=&quot;groupBy&quot;;\n      " +
+          "6 [label=&quot;ShuffledRDD [6]"))
+      }
     }
   }
 
diff --git a/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphListenerSuite.scala b/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphListenerSuite.scala
deleted file mode 100644
index 3fb78da0c7476..0000000000000
--- a/core/src/test/scala/org/apache/spark/ui/scope/RDDOperationGraphListenerSuite.scala
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.ui.scope
-
-import org.apache.spark.{SparkConf, SparkFunSuite}
-import org.apache.spark.scheduler._
-
-/**
- * Tests that this listener populates and cleans up its data structures properly.
- */
-class RDDOperationGraphListenerSuite extends SparkFunSuite {
-  private var jobIdCounter = 0
-  private var stageIdCounter = 0
-  private val maxRetainedJobs = 10
-  private val maxRetainedStages = 10
-  private val conf = new SparkConf()
-    .set("spark.ui.retainedJobs", maxRetainedJobs.toString)
-    .set("spark.ui.retainedStages", maxRetainedStages.toString)
-
-  test("run normal jobs") {
-    val startingJobId = jobIdCounter
-    val startingStageId = stageIdCounter
-    val listener = new RDDOperationGraphListener(conf)
-    assert(listener.jobIdToStageIds.isEmpty)
-    assert(listener.jobIdToSkippedStageIds.isEmpty)
-    assert(listener.stageIdToJobId.isEmpty)
-    assert(listener.stageIdToGraph.isEmpty)
-    assert(listener.completedStageIds.isEmpty)
-    assert(listener.jobIds.isEmpty)
-    assert(listener.stageIds.isEmpty)
-
-    // Run a few jobs, but not enough for clean up yet
-    (1 to 3).foreach { numStages => startJob(numStages, listener) } // start 3 jobs and 6 stages
-    (0 to 5).foreach { i => endStage(startingStageId + i, listener) } // finish all 6 stages
-    (0 to 2).foreach { i => endJob(startingJobId + i, listener) } // finish all 3 jobs
-
-    assert(listener.jobIdToStageIds.size === 3)
-    assert(listener.jobIdToStageIds(startingJobId).size === 1)
-    assert(listener.jobIdToStageIds(startingJobId + 1).size === 2)
-    assert(listener.jobIdToStageIds(startingJobId + 2).size === 3)
-    assert(listener.jobIdToSkippedStageIds.size === 3)
-    assert(listener.jobIdToSkippedStageIds.values.forall(_.isEmpty)) // no skipped stages
-    assert(listener.stageIdToJobId.size === 6)
-    assert(listener.stageIdToJobId(startingStageId) === startingJobId)
-    assert(listener.stageIdToJobId(startingStageId + 1) === startingJobId + 1)
-    assert(listener.stageIdToJobId(startingStageId + 2) === startingJobId + 1)
-    assert(listener.stageIdToJobId(startingStageId + 3) === startingJobId + 2)
-    assert(listener.stageIdToJobId(startingStageId + 4) === startingJobId + 2)
-    assert(listener.stageIdToJobId(startingStageId + 5) === startingJobId + 2)
-    assert(listener.stageIdToGraph.size === 6)
-    assert(listener.completedStageIds.size === 6)
-    assert(listener.jobIds.size === 3)
-    assert(listener.stageIds.size === 6)
-  }
-
-  test("run jobs with skipped stages") {
-    val startingJobId = jobIdCounter
-    val startingStageId = stageIdCounter
-    val listener = new RDDOperationGraphListener(conf)
-
-    // Run a few jobs, but not enough for clean up yet
-    // Leave some stages unfinished so that they are marked as skipped
-    (1 to 3).foreach { numStages => startJob(numStages, listener) } // start 3 jobs and 6 stages
-    (4 to 5).foreach { i => endStage(startingStageId + i, listener) } // finish only last 2 stages
-    (0 to 2).foreach { i => endJob(startingJobId + i, listener) } // finish all 3 jobs
-
-    assert(listener.jobIdToSkippedStageIds.size === 3)
-    assert(listener.jobIdToSkippedStageIds(startingJobId).size === 1)
-    assert(listener.jobIdToSkippedStageIds(startingJobId + 1).size === 2)
-    assert(listener.jobIdToSkippedStageIds(startingJobId + 2).size === 1) // 2 stages not skipped
-    assert(listener.completedStageIds.size === 2)
-
-    // The rest should be the same as before
-    assert(listener.jobIdToStageIds.size === 3)
-    assert(listener.jobIdToStageIds(startingJobId).size === 1)
-    assert(listener.jobIdToStageIds(startingJobId + 1).size === 2)
-    assert(listener.jobIdToStageIds(startingJobId + 2).size === 3)
-    assert(listener.stageIdToJobId.size === 6)
-    assert(listener.stageIdToJobId(startingStageId) === startingJobId)
-    assert(listener.stageIdToJobId(startingStageId + 1) === startingJobId + 1)
-    assert(listener.stageIdToJobId(startingStageId + 2) === startingJobId + 1)
-    assert(listener.stageIdToJobId(startingStageId + 3) === startingJobId + 2)
-    assert(listener.stageIdToJobId(startingStageId + 4) === startingJobId + 2)
-    assert(listener.stageIdToJobId(startingStageId + 5) === startingJobId + 2)
-    assert(listener.stageIdToGraph.size === 6)
-    assert(listener.jobIds.size === 3)
-    assert(listener.stageIds.size === 6)
-  }
-
-  test("clean up metadata") {
-    val startingJobId = jobIdCounter
-    val startingStageId = stageIdCounter
-    val listener = new RDDOperationGraphListener(conf)
-
-    // Run many jobs and stages to trigger clean up
-    (1 to 10000).foreach { i =>
-      // Note: this must be less than `maxRetainedStages`
-      val numStages = i % (maxRetainedStages - 2) + 1
-      val startingStageIdForJob = stageIdCounter
-      val jobId = startJob(numStages, listener)
-      // End some, but not all, stages that belong to this job
-      // This is to ensure that we have both completed and skipped stages
-      (startingStageIdForJob until stageIdCounter)
-        .filter { i => i % 2 == 0 }
-        .foreach { i => endStage(i, listener) }
-      // End all jobs
-      endJob(jobId, listener)
-    }
-
-    // Ensure we never exceed the max retained thresholds
-    assert(listener.jobIdToStageIds.size <= maxRetainedJobs)
-    assert(listener.jobIdToSkippedStageIds.size <= maxRetainedJobs)
-    assert(listener.stageIdToJobId.size <= maxRetainedStages)
-    assert(listener.stageIdToGraph.size <= maxRetainedStages)
-    assert(listener.completedStageIds.size <= maxRetainedStages)
-    assert(listener.jobIds.size <= maxRetainedJobs)
-    assert(listener.stageIds.size <= maxRetainedStages)
-
-    // Also ensure we're actually populating these data structures
-    // Otherwise the previous group of asserts will be meaningless
-    assert(listener.jobIdToStageIds.nonEmpty)
-    assert(listener.jobIdToSkippedStageIds.nonEmpty)
-    assert(listener.stageIdToJobId.nonEmpty)
-    assert(listener.stageIdToGraph.nonEmpty)
-    assert(listener.completedStageIds.nonEmpty)
-    assert(listener.jobIds.nonEmpty)
-    assert(listener.stageIds.nonEmpty)
-
-    // Ensure we clean up old jobs and stages, not arbitrary ones
-    assert(!listener.jobIdToStageIds.contains(startingJobId))
-    assert(!listener.jobIdToSkippedStageIds.contains(startingJobId))
-    assert(!listener.stageIdToJobId.contains(startingStageId))
-    assert(!listener.stageIdToGraph.contains(startingStageId))
-    assert(!listener.completedStageIds.contains(startingStageId))
-    assert(!listener.stageIds.contains(startingStageId))
-    assert(!listener.jobIds.contains(startingJobId))
-  }
-
-  test("fate sharing between jobs and stages") {
-    val startingJobId = jobIdCounter
-    val startingStageId = stageIdCounter
-    val listener = new RDDOperationGraphListener(conf)
-
-    // Run 3 jobs and 8 stages, finishing all 3 jobs but only 2 stages
-    startJob(5, listener)
-    startJob(1, listener)
-    startJob(2, listener)
-    (0 until 8).foreach { i => startStage(i + startingStageId, listener) }
-    endStage(startingStageId + 3, listener)
-    endStage(startingStageId + 4, listener)
-    (0 until 3).foreach { i => endJob(i + startingJobId, listener) }
-
-    // First, assert the old stuff
-    assert(listener.jobIdToStageIds.size === 3)
-    assert(listener.jobIdToSkippedStageIds.size === 3)
-    assert(listener.stageIdToJobId.size === 8)
-    assert(listener.stageIdToGraph.size === 8)
-    assert(listener.completedStageIds.size === 2)
-
-    // Cleaning the third job should clean all of its stages
-    listener.cleanJob(startingJobId + 2)
-    assert(listener.jobIdToStageIds.size === 2)
-    assert(listener.jobIdToSkippedStageIds.size === 2)
-    assert(listener.stageIdToJobId.size === 6)
-    assert(listener.stageIdToGraph.size === 6)
-    assert(listener.completedStageIds.size === 2)
-
-    // Cleaning one of the stages in the first job should clean that job and all of its stages
-    // Note that we still keep around the last stage because it belongs to a different job
-    listener.cleanStage(startingStageId)
-    assert(listener.jobIdToStageIds.size === 1)
-    assert(listener.jobIdToSkippedStageIds.size === 1)
-    assert(listener.stageIdToJobId.size === 1)
-    assert(listener.stageIdToGraph.size === 1)
-    assert(listener.completedStageIds.size === 0)
-  }
-
-  /** Start a job with the specified number of stages. */
-  private def startJob(numStages: Int, listener: RDDOperationGraphListener): Int = {
-    assert(numStages > 0, "I will not run a job with 0 stages for you.")
-    val stageInfos = (0 until numStages).map { _ =>
-      val stageInfo = new StageInfo(stageIdCounter, 0, "s", 0, Seq.empty, Seq.empty, "d")
-      stageIdCounter += 1
-      stageInfo
-    }
-    val jobId = jobIdCounter
-    listener.onJobStart(new SparkListenerJobStart(jobId, 0, stageInfos))
-    // Also start all stages that belong to this job
-    stageInfos.map(_.stageId).foreach { sid => startStage(sid, listener) }
-    jobIdCounter += 1
-    jobId
-  }
-
-  /** Start the stage specified by the given ID. */
-  private def startStage(stageId: Int, listener: RDDOperationGraphListener): Unit = {
-    val stageInfo = new StageInfo(stageId, 0, "s", 0, Seq.empty, Seq.empty, "d")
-    listener.onStageSubmitted(new SparkListenerStageSubmitted(stageInfo))
-  }
-
-  /** Finish the stage specified by the given ID. */
-  private def endStage(stageId: Int, listener: RDDOperationGraphListener): Unit = {
-    val stageInfo = new StageInfo(stageId, 0, "s", 0, Seq.empty, Seq.empty, "d")
-    listener.onStageCompleted(new SparkListenerStageCompleted(stageInfo))
-  }
-
-  /** Finish the job specified by the given ID. */
-  private def endJob(jobId: Int, listener: RDDOperationGraphListener): Unit = {
-    listener.onJobEnd(new SparkListenerJobEnd(jobId, 0, JobSucceeded))
-  }
-
-}