Update db-bench (#545)

* default to db-bench.csv

* remove unused --throughputOutput

* rename --timingOutput to --out

* exclude key/value generation from measured time

* use a multiplot that takes up to 5 csv files as input

* gitignore benchmark artifacts

.gitignore

@@ -16,3 +16,6 @@ npm-debug.log
 prebuilds/
 yarn.lock
 .nyc_output
+bench/db/
+bench/*.png
+*.csv

bench/README.md

@@ -0,0 +1,35 @@
+# Benchmarks
+
+## `db-bench`
+
+Run the benchmark:
+
+```
+node bench/db-bench.js
+```
+
+This writes results to `bench/db-bench.csv` by default. To save results from multiple runs, add `--out <filename>`. For example, you could switch branches (assuming that both branches have the same benchmark code):
+
+```
+node bench/db-bench.js --out master.csv
+
+git checkout wip
+node-gyp rebuild
+node bench/db-bench.js --out wip.csv
+```
+
+Then plot both runs with `db-bench-plot.sh <file, ..>` (the basenames of the files are used for the legend):
+
+```
+./bench/db-bench-plot.sh *.csv
+```
+
+If you are on Windows, you can use git bash:
+
+```
+bash bench/db-bench-plot.sh master.csv wip.csv
+```
+
+## `write-random`
+## `write-sorted`
+## `memory`

bench/db-bench-plot.sh

@@ -1,56 +1,46 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
+num_writes=1M
+num_files=$#
+platform=$(node -p process.platform)
+arch=$(node -p process.arch)
+node=$(node -p process.version)
+png_suffix=$(date '+%Y%m%d-%H.%M.%S')
 
 gnuplot <<EOF
   reset
-  set terminal pngcairo truecolor enhanced font "Ubuntu Mono,13" size 1920, 1080
-  set output "/tmp/5mbench.png"
+  set terminal pngcairo truecolor enhanced font "Ubuntu Mono,13" size 1920, 1080 background rgb "#1b1b1b"
+  set output "bench/db-bench-$png_suffix.png"
   set datafile separator ','
 
-  set logscale y
-  set nologscale y2
-  unset log y2
   set autoscale y
-  set autoscale y2
-  set ytics nomirror
-  set y2tics
-  set tics out
-
-  set xlabel "Minutes" tc rgb "#777777"
-  set ylabel "Milliseconds per write" tc rgb "#777777"
-  set y2label "Throughput MB/s" tc rgb "#777777"
+  set ytics mirror
+  set tics in
+  set xlabel "Time (seconds)" tc rgb "#777777"
 
-  set title "Node.js LevelDB (LevelDOWN): 100,000,000 random writes, 64M write buffer, HDD RAID1" tc rgb "#777777"
-  set key left tc rgb "#777777"
+  set key outside tc rgb "#777777"
   set border lc rgb "#777777"
 
-  set style line 1 lt 7 ps 1.2 lc rgb "#55019FD7"
-  set style line 2 lt 7 ps 0.1 lc rgb "#55019FD7"
-  set style line 3 lt 1 lw 2   lc rgb "#55019FD7"
-
-  set style line 4 lt 7 ps 1.2 lc rgb "#559ECC3C"
-  set style line 5 lt 7 ps 0.1 lc rgb "#559ECC3C"
-  set style line 6 lt 1 lw 2   lc rgb "#559ECC3C"
-
-  set style line 7 lt 7 ps 1.2 lc rgb "#55CC3C3C"
-  set style line 8 lt 7 ps 0.1 lc rgb "#55CC3C3C"
-  set style line 9 lt 1 lw 2   lc rgb "#55CC3C3C"
-
-  set style line 10 lt 7 ps 1.2 lc rgb "#553C3C3C"
-  set style line 11 lt 7 ps 0.1 lc rgb "#553C3C3C"
-  set style line 12 lt 1 lw 2   lc rgb "#553C3C3C"
-
-  plot \
-      1/0 with points title "Google LevelDB" ls 1 \
-    , 1/0 with points title "Hyper LevelDB"  ls 4 \
-    , 1/0 with points title "Basho LevelDB"  ls 7 \
-    , 1/0 with points title "LMDB"  ls 10 \
-    , "5m_google.csv" using (\$1/1000/60):(\$4/1000000) notitle         ls 2 axes x1y1 \
-    , "5m_hyper.csv"  using (\$1/1000/60):(\$4/1000000) notitle         ls 5 axes x1y1 \
-    , "5m_basho.csv"  using (\$1/1000/60):(\$4/1000000) notitle         ls 8 axes x1y1 \
-    , "5m_lmdb.csv"   using (\$1/1000/60):(\$4/1000000) notitle         ls 11 axes x1y1 \
-    , "5m_google.csv" using (\$1/1000/60):(\$5)         w lines notitle ls 3 axes x1y2 \
-    , "5m_hyper.csv"  using (\$1/1000/60):(\$5)         w lines notitle ls 6 axes x1y2 \
-    , "5m_basho.csv"  using (\$1/1000/60):(\$5)         w lines notitle ls 9 axes x1y2 \
-    , "5m_lmdb.csv"   using (\$1/1000/60):(\$5)         w lines notitle ls 12 axes x1y2 \
-
+  # To plot more than 5 files, add more line styles
+  set style line 1 lt 7 ps 0.8 lc rgb "#00FFFF"
+  set style line 2 lt 7 ps 0.8 lc rgb "#D84797"
+  set style line 3 lt 7 ps 0.8 lc rgb "#23CE6B"
+  set style line 4 lt 7 ps 0.8 lc rgb "#F5B700"
+  set style line 5 lt 7 ps 0.8 lc rgb "#731DD8"
+
+  filename(n) = word("$@", n)
+  basename(n) = substr(word("$@", n), 0, strstrt(word("$@", n), ".csv") - 1)
+
+  set multiplot layout 2,1
+    set lmargin at screen 0.1
+    set title "leveldown db-bench, $num_writes writes, $platform $arch, node $node" tc rgb "#cccccc"
+    set ylabel "Milliseconds/write" tc rgb "#888888"
+    set logscale y
+    plot for [i=1:$num_files] filename(i) using (\$1/1000):(\$4/1000000) title basename(i) ls i axes x1y1
+
+    set title ""
+    set ylabel "Throughput MB/s" tc rgb "#888888"
+    set nologscale y
+    plot for [i=1:$num_files] filename(i) using (\$1/1000):(\$5) w lines title basename(i) ls i axes x1y1
+  unset multiplot
 EOF

bench/db-bench.js

@@ -15,8 +15,7 @@ const options = {
   cacheSize: argv.cacheSize || 8,
   writeBufferSize: argv.writeBufferSize || 4,
   valueSize: argv.valueSize || 100,
-  timingOutput: argv.timingOutput || path.join(__dirname, 'timingOutput'),
-  throughputOutput: argv.throughputOutput
+  out: argv.out || path.join(__dirname, 'db-bench.csv')
 }
 
 const randomString = require('slump').string
@@ -27,7 +26,7 @@ if (!options.useExisting) {
 }
 
 const db = leveldown(options.db)
-const timesStream = fs.createWriteStream(options.timingOutput, 'utf8')
+const timesStream = fs.createWriteStream(options.out, 'utf8')
 
 function make16CharPaddedKey () {
   const r = Math.floor(Math.random() * options.num)
@@ -87,13 +86,18 @@ function start () {
       timesAccum = 0
     }
 
-    var time = process.hrtime()
+    var key = make16CharPaddedKey()
+    var value = randomString({ length: options.valueSize })
+    var start = process.hrtime()
 
-    db.put(make16CharPaddedKey(), randomString({ length: options.valueSize }), function (err) {
+    db.put(key, value, function (err) {
       if (err) throw err
 
+      var duration = process.hrtime(start)
+      var nano = (duration[0] * 1e9) + duration[1]
+
       totalBytes += keyTmpl.length + options.valueSize
-      timesAccum += process.hrtime(time)[1]
+      timesAccum += nano
       inProgress--
       process.nextTick(write)
     })