From fc074a322809572410383b954a32e360fbfe4160 Mon Sep 17 00:00:00 2001
From: David Baker Effendi <dave@whirlylabs.com>
Date: Tue, 13 Aug 2024 20:23:50 +0200
Subject: [PATCH] Added check to disregard jmh failed attempts as legit

---
 runBenchmarks.sc | 49 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/runBenchmarks.sc b/runBenchmarks.sc
index d222181d..e6e88575 100644
--- a/runBenchmarks.sc
+++ b/runBenchmarks.sc
@@ -10,8 +10,7 @@ import scala.util.boundary
 import scala.util.boundary.break
 
 // Combinations of driver, project, Gb mem, known to OOM
-val oomCombinations: Set[(String, String, Int)] = Set(("tinkergraph", "compress", 2))
-val drivers                                     = Seq("overflowdb", "tinkergraph", "neo4j-embedded")
+val drivers = Seq("overflowdb", "tinkergraph", "neo4j-embedded")
 
 @main def main(): Unit = {
   println("[info] Ensuring compilation status and benchmark dataset availability...")
@@ -28,7 +27,9 @@ val drivers                                     = Seq("overflowdb", "tinkergraph
     val outputPath       = Path.of(driverResultsDir.toString, s"output-Xmx${memGb}G")
     val (writeOutputFile, readOutputFile) =
       (Path.of(s"$outputPath-write.txt").toFile, Path.of(s"$outputPath-read.txt").toFile)
-    val existingAttempt = Path.of(s"$outputPath-sbt.txt").toFile.exists()
+    val sbtFile = Path.of(s"$outputPath-sbt.txt").toFile
+    val existingAttempt =
+      sbtFile.exists() && !readLogsForJmhClassLoaderError(sbtFile) // If there is a jmh error we retry
     val cmd =
       s"Jmh/runMain com.github.plume.oss.Benchmark $driver ${projectDir.toAbsolutePath} -o ${outputPath.toAbsolutePath} -r ${resultsPath.toAbsolutePath} -m $memGb"
     JmhProcessInfo(cmd, existingAttempt, writeOutputFile, readOutputFile)
@@ -47,27 +48,40 @@ val drivers                                     = Seq("overflowdb", "tinkergraph
     drivers.foreach { driver =>
       projects.foreach { project =>
         val projectName = project.getFileName.toString.toLowerCase.stripSuffix(".jar")
-        if (oomCombinations.contains(driver, projectName, memConfig)) {
+        val JmhProcessInfo(cmd, resultsExist, writeOutputFile, readOutputFile) =
+          benchmarkArgs(driver, project.getFileName.toString, memConfig)
+        if (resultsExist) {
           println(
-            s"[info] '$driver' on project '$project' with `-Xmx${memConfig}G` will cause an OutOfMemoryException. Skipping..."
+            s"[info] An attempt for '$driver' on project '$projectName' with `-Xmx${memConfig}G` already exist. Skipping..."
           )
         } else {
-          val JmhProcessInfo(cmd, resultsExist, writeOutputFile, readOutputFile) =
-            benchmarkArgs(driver, project.getFileName.toString, memConfig)
-          if (resultsExist) {
-            println(
-              s"[info] An attempt for '$driver' on project '$project' with `-Xmx${memConfig}G` already exist. Skipping..."
-            )
-          } else {
-            println(s"[info] Benchmarking '$driver' on project '$project' with `-Xmx${memConfig}G`")
-            runAndMonitorBenchmarkProcess(cmd, writeOutputFile, readOutputFile)
-          }
+          println(s"[info] Benchmarking '$driver' on project '$projectName' with `-Xmx${memConfig}G`")
+          runAndMonitorBenchmarkProcess(cmd, writeOutputFile, readOutputFile)
         }
       }
     }
   }
 }
 
+def readLogsForJmhClassLoaderError(file: File): Boolean = {
+  val reader   = new BufferedReader(new FileReader(file))
+  var hasError = false
+  try {
+    var line: String = null
+    while ({
+      line = reader.readLine();
+      line != null
+    }) {
+      if (line.contains("jmhType.class (No such file or directory)")) {
+        hasError = true
+      }
+    }
+  } finally {
+    reader.close()
+  }
+  hasError
+}
+
 def sendCtrlCSignal(processId: Long): Unit = {
   val osName = System.getProperty("os.name").toLowerCase
   if (osName.contains("win")) {
@@ -83,7 +97,7 @@ def runAndMonitorBenchmarkProcess(cmd: String, writeOutputFile: File, readOutput
   writeOutputFile.createIfNotExists
   readOutputFile.createIfNotExists
 
-  val sbtFile = File(writeOutputFile.getAbsolutePath.stripSuffix("write.txt") + "sbt.txt")
+  val sbtFile        = File(writeOutputFile.getAbsolutePath.stripSuffix("write.txt") + "sbt.txt")
   val processBuilder = new java.lang.ProcessBuilder("sbt", cmd).redirectOutput(sbtFile)
 
   // Ignore locks for aborted JMH processes
@@ -123,7 +137,8 @@ def runAndMonitorBenchmarkProcess(cmd: String, writeOutputFile: File, readOutput
   var shouldTerminate = false
   while (!shouldTerminate && process.isAlive) {
     Thread.sleep(5000)
-    shouldTerminate = readLogsForErrors(writeOutputFile) || readLogsForErrors(readOutputFile) || readLogsForErrors(sbtFile)
+    shouldTerminate =
+      readLogsForErrors(writeOutputFile) || readLogsForErrors(readOutputFile) || readLogsForErrors(sbtFile)
   }
 }