Skip to content

Commit

Permalink
[SPARK-24414][ui] Calculate the correct number of tasks for a stage.
Browse files Browse the repository at this point in the history
This change takes into account all non-pending tasks when calculating
the number of tasks to be shown. This also means that when the stage
is pending, the task table (or, in fact, most of the data in the stage
page) will not be rendered.

I also fixed the label when the known number of tasks is larger than
the recorded number of tasks (it was inverted).
  • Loading branch information
Marcelo Vanzin committed May 29, 2018
1 parent a53ea70 commit 40b6cb7
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We

val localitySummary = store.localitySummary(stageData.stageId, stageData.attemptId)

val totalTasks = stageData.numActiveTasks + stageData.numCompleteTasks +
stageData.numFailedTasks + stageData.numKilledTasks
val totalTasks = taskCount(stageData)
if (totalTasks == 0) {
val content =
<div>
Expand All @@ -133,7 +132,7 @@ private[ui] class StagePage(parent: StagesTab, store: AppStatusStore) extends We
val totalTasksNumStr = if (totalTasks == storedTasks) {
s"$totalTasks"
} else {
s"$totalTasks, showing ${storedTasks}"
s"$storedTasks, showing ${totalTasks}"
}

val summary =
Expand Down Expand Up @@ -686,7 +685,7 @@ private[ui] class TaskDataSource(

private var _tasksToShow: Seq[TaskData] = null

override def dataSize: Int = stage.numTasks
override def dataSize: Int = taskCount(stage)

override def sliceData(from: Int, to: Int): Seq[TaskData] = {
if (_tasksToShow == null) {
Expand Down Expand Up @@ -1052,4 +1051,9 @@ private[ui] object ApiHelper {
(stage.map(_.name).getOrElse(""), stage.flatMap(_.description).getOrElse(job.name))
}

def taskCount(stageData: StageData): Int = {
stageData.numActiveTasks + stageData.numCompleteTasks + stageData.numFailedTasks +
stageData.numKilledTasks
}

}

0 comments on commit 40b6cb7

Please sign in to comment.