From 83306a8bc1749974fd60edf441843ea9a93c3520 Mon Sep 17 00:00:00 2001 From: Joshi Date: Sun, 12 Jul 2015 12:27:27 -0700 Subject: [PATCH] History Server: updated order for multiple attempts(descending start time) --- .../org/apache/spark/deploy/history/FsHistoryProvider.scala | 6 +++--- .../spark/deploy/history/FsHistoryProviderSuite.scala | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index 250c38228ad7d..051b5e40d20b4 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -413,8 +413,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) /** * Comparison function that defines the sort order for application attempts within the same - * application. Order is: completed attempts before running attempts, running attempts sorted - * by ascending start time,completed attempts sorted by ascending end time. + * application. Order is: completed attempts before running attempts, if both completed + * or both running attempts sorted by descending start time. * * Normally applications should have a single running attempt; but failure to call sc.stop() * may cause multiple running attempts to show up. @@ -425,7 +425,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) a1: FsApplicationAttemptInfo, a2: FsApplicationAttemptInfo): Boolean = { if (a1.completed == a2.completed) { - if (a1.completed) a1.endTime <= a2.endTime else a1.startTime <= a2.startTime + a1.startTime >= a2.startTime } else { a1.completed } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala index b9511eb8b82b9..12ec126832b04 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala @@ -272,7 +272,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc updateAndCheck(provider) { list => list.size should be (1) list.head.attempts.size should be (2) - list.head.attempts.head.attemptId should be (Some("attempt1")) + list.head.attempts.head.attemptId should be (Some("attempt2")) } val attempt3 = newLogFile("app1", Some("attempt3"), inProgress = false) @@ -289,7 +289,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc } val app2Attempt1 = newLogFile("app2", Some("attempt1"), inProgress = false) - writeFile(attempt2, true, None, + writeFile(attempt1, true, None, SparkListenerApplicationStart("app2", Some("app2"), 5L, "test", Some("attempt1")), SparkListenerApplicationEnd(6L) )