Merge pull request #86 from weymouth/benchmarks

General benchmarking framework

.gitignore

@@ -3,6 +3,7 @@
 *.gif
 *.mp4
 *.dat
+*.json
 *.pdf
 *.vti
 *.pvd

benchmark/Project.toml

@@ -0,0 +1,6 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c"
+PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+WaterLily = "ed894a53-35f9-47f1-b17f-85db9237eebd"

benchmark/README.md

@@ -0,0 +1,45 @@
+# Automatic benchmark generation suite
+
+Suite to generate benchmarks across different Julia versions (using [juliaup](https://github.com/JuliaLang/juliaup)), backends, cases, and cases sizes using the [benchmark.sh](./benchmark.sh) script.
+
+## TL;DR
+Usage example
+```
+sh benchmark.sh  -v "1.9.4 1.10.0-rc1" -t "1 3 6" -b "Array CuArray" -c "tgv.jl" -p "5,6,7"
+```
+The default launch is equivalent to:
+```
+sh benchmark.sh  -v JULIA_USER_VERSION -t "1 6" -b "Array CuArray" -c "tgv.jl" -p "5,6,7" -s 100 -ft Float32
+```
+
+## Usage information
+
+The accepted command line arguments are (parenthesis for short version):
+ - Backend arguments: `--version(-v)`, `--backends(-b)`, `--threads(-t)`. Respectively: Julia version, backend types, number of threads (for Array backend). These arguments accept a list of different parameters, for example:
+    ```
+    -v "1.8.5 1.9.4" -b "Array CuArray" -t "1 6"
+    ```
+    which would generate benchmark for all these combinations of parameters.
+ - Case arguments: `--cases(-c)`, `--log2p(-p)`, `--max_steps(-s)`, `--ftype(-ft)`. Respectively: Benchmark case file, case sizes, number of time steps, float data type. The following arguments would generate benchmarks for the [`tgv.jl`](./tgv.jl) case:
+    ```
+    -c "tgv.jl" -p "5,6,7" -s 100 -ft "Float32"
+    ```
+    which in addition to the benchmark arguments, altogether can be used to launch this script as:
+    ```
+    sh benchmark.sh -v "1.8.5 1.9.4" -b "Array CuArray" -t "1 6" -c "tgv.jl" -p "5,6,7" -s 100 -ft "Float32"
+    ```
+    Case arguments accept a list of parameters for each case, and the list index is shared across these arguments (hence lists must have equal length):
+    ```
+    -c "tgv.jl donut.jl" -p "5,6,7 7,8" -s "100 500" -ft "Float32 Float64"
+    ```
+    which would run the same benchmarks for the TGV as before, and benchmarks for the donut case too resulting into 2 Julia versions x (2 Array + 1 CuArray) backends x (3 TGV sizes + 2 donut sizes) = 30 benchmarks.
+
+Benchmarks are saved in JSON format with the following nomenclature: `casename_sizes_maxsteps_ftype_backend_waterlilyHEADhash_juliaversion.json`. Benchmarks can be finally compared using [`compare.jl`](./compare.jl) as follows
+```
+julia --project compare.jl benchmark_1.json benchmark_2.json benchmark_3.json ...
+```
+Note that each case benchmarks should be compared separately. If a single case is benchmarked, and all the JSON files in the current directory belong to it, one can simply run:
+```
+julia --project compare.jl $(find . -name "*.json" -printf "%T@ %Tc %p\n" | sort -n | awk '{print $8}')
+```
+which would take all the JSON files, sort them by creation time, and pass them as arguments to the `compare.jl` program. Finally, note that the first benchmark passed as argument is taken as reference to compute speedups of other benchmarks: `speedup_x = time(benchmark_1) / time(benchmark_x)`.

benchmark/benchmark.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+# Grep current julia version
+julia_version () {
+    julia_v=($(julia -v))
+    echo "${julia_v[2]}"
+}
+
+# Update project environment with new Julia version
+update_environment () {
+    echo "Updating environment to Julia v$version"
+    # Mark WaterLily as a development package. Then update dependencies and precompile.
+    julia +${version} --project -e "using Pkg; Pkg.develop(PackageSpec(path=dirname(@__DIR__))); Pkg.update();"
+}
+
+run_benchmark () {
+    echo "Running: julia +${version} --project --startup-file=no $args"
+    julia +${version} --project --startup-file=no $args
+}
+
+# Print benchamrks info
+display_info () {
+    echo "--------------------------------------"
+    echo "Running benchmark tests for:
+ - Julia:        ${VERSIONS[@]}
+ - Backends:     ${BACKENDS[@]}"
+    if [[ " ${BACKENDS[*]} " =~ [[:space:]]'Array'[[:space:]] ]]; then
+        echo " - CPU threads:  ${THREADS[@]}"
+    fi
+    echo " - Cases:        ${CASES[@]}
+ - Size:         ${LOG2P[@]:0:$NCASES}
+ - Sim. steps:   ${MAXSTEPS[@]:0:$NCASES}
+ - Data type:    ${FTYPE[@]:0:$NCASES}"
+    echo "--------------------------------------"; echo
+}
+
+# Default backends
+JULIA_USER_VERSION=$(julia_version)
+VERSION=($JULIA_USER_VERSION)
+BACKENDS=('Array' 'CuArray')
+THREADS=('1' '6')
+# Default cases. Arrays below must be same length (specify each case individually)
+CASES=('tgv.jl')
+LOG2P=('5,6,7')
+MAXSTEPS=('100')
+FTYPE=('Float32')
+
+# Parse arguments
+while [ $# -gt 0 ]; do
+case "$1" in
+    --versions|-v)
+    VERSIONS=($2)
+    shift
+    ;;
+    --backends|-b)
+    BACKENDS=($2)
+    shift
+    ;;
+    --threads|-t)
+    THREADS=($2)
+    shift
+    ;;
+    --cases|-c)
+    CASES=($2)
+    shift
+    ;;
+    --log2p|-p)
+    LOG2P=($2)
+    shift
+    ;;
+    --max_steps|-s)
+    MAXSTEPS=($2)
+    shift
+    ;;
+    --float_type|-ft)
+    FTYPE=($2)
+    shift
+    ;;
+    *)
+    printf "ERROR: Invalid argument %s\n" "${1}" 1>&2
+    exit 1
+esac
+shift
+done
+
+NCASES=${#CASES[@]}
+
+# Assert "--threads" argument is not empy if "Array" backend is present
+if [[ " ${BACKENDS[*]} " =~ [[:space:]]'Array'[[:space:]] ]]; then
+    if [ "${#THREADS[@]}" == 0 ]; then
+        echo "ERROR: Backend 'Array' is present, but '--threads' argument is empty."
+        exit 1
+    fi
+fi
+
+# Display information
+display_info
+
+# Benchmarks
+for version in "${VERSIONS[@]}" ; do
+    echo "Julia v$version benchmaks"
+    update_environment
+    for i in "${!CASES[@]}"; do
+        args_case="${CASES[$i]} --log2p="${LOG2P[$i]}" --max_steps=${MAXSTEPS[$i]} --ftype=${FTYPE[$i]}"
+        for backend in "${BACKENDS[@]}" ; do
+            if [ "${backend}" == "Array" ]; then
+                for thread in "${THREADS[@]}" ; do
+                    args="-t $thread "$args_case" --backend=$backend"
+                    run_benchmark
+                done
+            else
+                args=$args_case" --backend=$backend"
+                run_benchmark
+            fi
+        done
+    done
+done
+
+echo "All done!"
+exit 0

benchmark/compare.jl

@@ -0,0 +1,25 @@
+using BenchmarkTools, PrettyTables
+
+# Load benchmarks
+benchmarks = [BenchmarkTools.load(f)[1] for f in ARGS]
+# Get backends string vector and assert same case sizes for the different backends
+backends_str = [String.(k)[1] for k in keys.(benchmarks)]
+log2p_str = [String.(keys(benchmarks[i][backend_str])) for (i, backend_str) in enumerate(backends_str)]
+@assert length(unique(log2p_str)) == 1
+# Assuming the case and tested function is the same in all benchmarks, we grab their name
+case, f_test = benchmarks[1].tags[1:2]
+# Get data for PrettyTables
+header = ["Backend", "WaterLily", "Julia", "Precision", "Allocations", "GC [%]", "Time [s]", "Speed-up"]
+data, base_speedup = Matrix{Any}(undef, length(benchmarks), length(header)), 1.0
+printstyled("Benchmark environment: $case $f_test (max_steps=$(benchmarks[1].tags[4]))\n", bold=true)
+for n in log2p_str[1]
+    printstyled("▶ log2p = $n\n", bold=true)
+    for (i, benchmark) in enumerate(benchmarks)
+        datap = benchmark[backends_str[i]][n][f_test]
+        speedup = i == 1 ? 1.0 : benchmarks[1][backends_str[1]][n][f_test].times[1] / datap.times[1]
+        data[i, :] .= [backends_str[i], benchmark.tags[end-1], benchmark.tags[end], benchmark.tags[end-3],
+            datap.allocs, (datap.gctimes[1] / datap.times[1]) * 100.0, datap.times[1] / 1e9, speedup]
+    end
+    pretty_table(data; header=header, header_alignment=:c, formatters=ft_printf("%.2f", [6,7,8]))
+end
+