Turn All.som into a benchmark that runs most other benchmarks (#122)

smarr · Aug 21, 2024 · f11313b · f11313b
2 parents 9c04914 + d1ed7da
commit f11313b
Show file tree

Hide file tree

Showing 2 changed files with 53 additions and 52 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -178,3 +178,10 @@ jobs:
           ${{ matrix.som }} -cp ../Smalltalk:../TestSuite ../Examples/Benchmarks/TestSuite/TestTestSuite.som
           ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som --gc TestGC100 1 1
           ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite ../Examples/Benchmarks/BenchmarkHarness.som Test100 1 1
+
+      - name: Test All Benchmark
+        # ignore VMs that give slighly different results for tests
+        if: ${{ matrix.repo != 'som-rs.git' && matrix.som != 'spec' && matrix.repo != 'JsSOM.git' && matrix.repo != 'PySOM.git' }}
+        run: |
+          cd som-vm
+          ${{ matrix.som }} -cp ../Smalltalk:../Examples/Benchmarks/TestSuite:../Examples/Benchmarks/Richards:../Examples/Benchmarks/DeltaBlue:../Examples/Benchmarks/NBody:../Examples/Benchmarks/Json:../Examples/Benchmarks/GraphSearch:../Examples/Benchmarks/LanguageFeatures ../Examples/Benchmarks/BenchmarkHarness.som All 1 1
diff --git a/Examples/Benchmarks/All.som b/Examples/Benchmarks/All.som
@@ -20,63 +20,57 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 "
 
-All = BenchmarkHarness (
-    | summedAverage |
+All = Benchmark (
+    oneTimeSetup = (
+        self all do: [:pair |
+            system load: (pair at: 1)
+        ]
+    )
 
     all = (
-        ^ Fibonacci, Dispatch, Bounce, Loop, Permute, Queens, List, Recurse,
-          Storage, Sieve, BubbleSort, QuickSort, Sum, Towers, TreeSort,
-          IntegerLoop, FieldLoop
-    )
-
-    run: params = (
-        params length < 2
-            ifTrue:  [ self exec: 100 ]
-            ifFalse: [ self exec: (params at: 2) asInteger ]
-    )
-
-    printUsage = (
-      './som.sh -cp Smalltalk Examples/Benchmarks/All.som [number-of-iterations]' println.
-      '' println.
-      '  number-of-iterations - the number of time each benchmark is executed, default: 1' println.
-    )
-
-    initialize = (
-        super initialize.
-        summedAverage := 0.
+        ^ #(#Richards 1),
+          #(#DeltaBlue 100),
+          #(#NBody 1000),
+          #(#Json 1),
+          #(#GraphSearch 7),
+          #(#PageRank 75),
+          #(#Fannkuch 7),
+          #(#Fibonacci 10),
+          #(#Dispatch 10),
+          #(#Bounce 10),
+          #(#Loop 100),
+          #(#Permute 10),
+          #(#Queens 10),
+          #(#List 2),
+          #(#Recurse 12),
+          #(#Storage 8),
+          #(#Sieve 20),
+          #(#BubbleSort 15),
+          #(#QuickSort 15),
+          #(#Sum 40),
+          #(#Towers 2),
+          #(#TreeSort 7),
+          #(#IntegerLoop 7),
+          #(#FieldLoop 1),
+          #(#WhileLoop 30),
+          #(#Mandelbrot 50),
+          #(#Test 1)
     )
-
-    exec: iterations = (
-        'Start execution of all benchmarks. Iterations: ' print.
-        iterations println.
-
-        self all do: [:cls |
-            self initialize.
-            self benchmarkClass: cls.
-            self printAll: false.
-            self maxRuntime: 3. "seconds"
-            self numIterations: iterations.
-            self warmUp: 10.
 
-            self runBenchmark.
+    benchmark = (
+        self all do: [:pair |
+            | benchName size benchClass bench |
+            benchName := pair at: 1.
+            size := pair at: 2.
+
+            benchClass := system load: benchName.
+            bench := benchClass new.
+            bench oneTimeSetup.
+            (bench innerBenchmarkLoop: size) ifFalse: [
+              self error: benchName + ' failed with incorrect result' ].
         ].
-        self printTotal.
+        ^ true
     )
-
-    reportBenchmark: bench result: total = (
-        '' println.
-        'Benchmark: ' print.
-        bench name println.
 
-        ('   Iterations: ' + numIterations + ' (elapsed time ' + (total // 1000) round
-            + ' ms)') println.
-        ('   AVERAGE: ' + ((total // numIterations) // 1000) round + ' ms') println.
-
-        summedAverage := summedAverage + (total // numIterations).
-    )
-
-    printTotal = (
-        ('Summed Average Runtime: ' + (summedAverage // 1000) round asString + ' ms') println.
-    )
-
+    verifyResult: result = ( ^ result )
 )