Publishing benchmarks for graphs (#740)

* [Do not merge] Publishing benchmarks for graphs

* add wasmtime-benchmark-master

* add dbg ref

* add collect artifacts

* switch ci image

* add publish

* cp instaed mv

* add gha

* fix on

* add gh-pages

* disable ref for checkout

* trim whitespaces

* add cancel for previous runs

* add gh token

* downgrade checkout action

* add skip-fetch-gh-pages option

* debug gha

* add timer for files

* fix script

* move script to file

* rename job

* restart pipeline

* add debug messages

* enable script

* restart pipeline to add second result to graph

* remove debug refs

---------

Co-authored-by: Robin Freyler <[email protected]>

.github/workflows/publish-benchmarks.yml

@@ -0,0 +1,62 @@
+name: Benchmarks - publish graphs
+
+on:
+  push:
+    branches: [master]
+
+permissions:
+  # deployments permission to deploy GitHub pages website
+  deployments: write
+  # contents permission to update benchmark contents in gh-pages branch
+  contents: write
+
+# disable running jobs on earlier commits
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    name: Publishing benchmarks graphs
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout gh-pages branch
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+        with:
+          ref: "gh-pages"
+
+      - name: Waiting for the output files
+        run: bash scripts/ci/sync_wait.sh
+
+      - name: Store benchmark result criterion
+        uses: benchmark-action/github-action-benchmark@70405016b032d44f409e4b1b451c40215cbe2393 # v1.18.0
+        with:
+          name: "Wasmi criterion benchmark"
+          tool: "cargo"
+          output-file-path: benchmarks/criterion/output-criterion.txt
+          benchmark-data-dir-path: "benchmarks/criterion"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          skip-fetch-gh-pages: true
+
+      - name: Store benchmark result wasmtime
+        uses: benchmark-action/github-action-benchmark@70405016b032d44f409e4b1b451c40215cbe2393 # v1.18.0
+        with:
+          name: "Wasmi criterion wasmtime"
+          tool: "cargo"
+          output-file-path: benchmarks/wasmtime/output-wasmtime.txt
+          benchmark-data-dir-path: "benchmarks/wasmtime"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          skip-fetch-gh-pages: true
+
+      - name: Push benchmark result
+        run: |
+          git config --global user.name 'github-action-benchmark'
+          git config --global user.email '[email protected]'
+          echo "removing files"
+          rm -f benchmarks/criterion/output-criterion.txt
+          rm -f benchmarks/wasmtime/output-wasmtime.txt
+          echo "git add and commit"
+          git add benchmarks/
+          git commit -m "remove output files"
+          echo "git push"
+          git push 'https://${{ secrets.GITHUB_TOKEN }}@github.com/paritytech/wasmi.git' gh-pages:gh-pages

.github/workflows/rust.yml

@@ -2,9 +2,14 @@ name: Rust - Continuous Integration
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
+
+# disable running jobs on earlier commits
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   check:
@@ -27,7 +32,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-            target/            
+            target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
       - name: Add extra targets
@@ -79,22 +84,22 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-            target/            
+            target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
       - name: Checkout Submodules
         run: git submodule update --init --recursive
       - name: Test (default features)
         uses: actions-rs/cargo@v1
         env:
-          RUSTFLAGS: '--cfg debug_assertions'
+          RUSTFLAGS: "--cfg debug_assertions"
         with:
           command: test
           args: --workspace --release
       - name: Test (all features)
         uses: actions-rs/cargo@v1
         env:
-          RUSTFLAGS: '--cfg debug_assertions'
+          RUSTFLAGS: "--cfg debug_assertions"
         with:
           command: test
           args: --workspace --release --all-features
@@ -135,12 +140,12 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-            target/            
+            target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
       - uses: actions-rs/cargo@v1
         env:
-          RUSTDOCFLAGS: '-D warnings'
+          RUSTDOCFLAGS: "-D warnings"
         with:
           command: doc
           args: --workspace --all-features --no-deps --document-private-items
@@ -158,7 +163,7 @@ jobs:
       - uses: actions-rs/cargo@v1
         with:
           command: audit
-          args: ''
+          args: ""
 
   udeps:
     name: uDeps
@@ -289,7 +294,7 @@ jobs:
             ~/.cargo/registry/index/
             ~/.cargo/registry/cache/
             ~/.cargo/git/db/
-            target/            
+            target/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
       - name: Clippy (default features)
@@ -343,7 +348,7 @@ jobs:
       - name: Run cargo-tarpaulin (default features)
         uses: actions-rs/[email protected]
         with:
-          version: '0.18.0'
+          version: "0.18.0"
           args: --workspace
       - name: Upload to codecov.io
         uses: codecov/[email protected]

.gitlab-ci.yml

@@ -8,40 +8,55 @@ default:
   retry:
     max: 2
 
-.docker-env:                       &docker-env
-  image:                           "paritytech/ci-linux:staging"
+.docker-env:
+  image:                           "paritytech/ci-unified:bullseye-1.71.0"
   interruptible:                   true
   tags:
     - benchmark
 
-.kubernetes-env:                   &kubernetes-env
+.kubernetes-env:
   image:                           "paritytech/tools:latest"
   tags:
     - kubernetes-parity-build
 
-# benchmarks
-criterion-benchmark:
-  stage:                           benchmarks
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                   # PRs
-  <<:                              *docker-env
+.collect-artifacts:
   artifacts:
     name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
     when:                          on_success
     expire_in:                     1 days
     paths:
       - target/ci/
-  script:
+
+.common-before:
+  before_script:
     # Show Versions
     - cargo --version
     - rustc --version
+    # Create dir for artifacts
+    - mkdir -p ./target/ci
+
+.pr-refs:
+  rules:
+    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                   # PRs
 
+.master-refs:
+  rules:
+    - if: $CI_COMMIT_REF_NAME == "master"
+
+# benchmarks
+criterion-benchmark:
+  stage:                           benchmarks
+  extends:
+    - .common-before
+    - .pr-refs
+    - .docker-env
+    - .collect-artifacts
+  script:
     - git fetch
     # Benchmark Branch: `master`
     - git checkout master
     - git pull
     - git submodule update --init --recursive
-    - mkdir -p ./target/ci
     - pushd ./crates/wasmi/
     - cargo bench --bench benches -- --noplot --save-baseline master | tee ../../target/ci/bench-master
     - popd
@@ -59,20 +74,12 @@ criterion-benchmark:
 
 wasmtime-benchmark:
   stage:                           benchmarks
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                   # PRs
-  <<:                              *docker-env
-  artifacts:
-    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
-    when:                          on_success
-    expire_in:                     1 days
-    paths:
-      - target/ci/
+  extends:
+    - .common-before
+    - .pr-refs
+    - .docker-env
+    - .collect-artifacts
   script:
-    # Show Versions
-    - cargo --version
-    - rustc --version
-
     # Install dependencies
     - cargo install cargo-wasi
     - curl https://wasmtime.dev/install.sh -sSf | bash
@@ -83,7 +90,6 @@ wasmtime-benchmark:
     - git checkout master
     - git pull
     - git submodule update --init --recursive
-    - mkdir -p ./target/ci
     - cargo wasi build --profile bench --bench benches --verbose
     - cp `ls -t target/wasm32-wasi/release/deps/*wasi.wasm` target/ci/benches-master.wasm
     - pushd ./crates/wasmi/
@@ -95,22 +101,83 @@ wasmtime-benchmark:
     # `wasmtime` Branch: `pr`
     - git checkout $CI_COMMIT_SHA
     - git submodule update --init --recursive
-    - mkdir -p ./target/ci
     - cargo wasi build --profile bench --bench benches --verbose
     - cp `ls -t target/wasm32-wasi/release/deps/*wasi.wasm` target/ci/benches-pr.wasm
     - pushd ./crates/wasmi/
     - wasmtime run --dir=. ../../target/ci/benches-pr.wasm -- --bench --baseline master-wasm | tee ../../target/ci/wasmtime-pr
     - popd
     - mv ./crates/wasmi/target/criterion ./target/ci/wasmtime-criterion
 
+criterion-benchmark-master:
+  stage:                           benchmarks
+  extends:
+    - .common-before
+    - .master-refs
+    - .docker-env
+    - .collect-artifacts
+  script:
+    - git submodule update --init --recursive
+    - cd ./crates/wasmi/
+    - cargo bench --bench benches -- --noplot --save-baseline master --output-format bencher | tee ../../target/ci/output-criterion.txt
+
+wasmtime-benchmark-master:
+  stage:                           benchmarks
+  extends:
+    - .common-before
+    - .master-refs
+    - .docker-env
+    - .collect-artifacts
+  script:
+    # Install dependencies
+    - cargo install cargo-wasi
+    - curl https://wasmtime.dev/install.sh -sSf | bash
+    - source ~/.profile
+    # `wasmtime` Branch: `master`
+    - git submodule update --init --recursive
+    - cargo wasi build --profile bench --bench benches --verbose
+    - cp `ls -t target/wasm32-wasi/release/deps/*wasi.wasm` target/ci/benches-master.wasm
+    - pushd ./crates/wasmi/
+    - wasmtime run --dir=. ../../target/ci/benches-master.wasm -- --bench --save-baseline master-wasm --output-format bencher | tee ../../target/ci/output-wasmtime.txt
+
+
 # report
 publish-report:
   stage:                           report
+  extends:
+    - .pr-refs
+    - .kubernetes-env
   needs:
-    - criterion-benchmark
-    - wasmtime-benchmark
-  rules:
-  - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                   # PRs
-  <<:                              *kubernetes-env
+    - job: criterion-benchmark
+      artifacts: true
+    - job: wasmtime-benchmark
+      artifacts: true
+  script:
+    - bash ./scripts/ci/benchmarks-report.sh
+
+publish-report-master:
+  stage:                           report
+  extends:
+    - .master-refs
+    - .kubernetes-env
+  needs:
+    - job: criterion-benchmark-master
+      artifacts: true
+    - job: wasmtime-benchmark-master
+      artifacts: true
   script:
-    - bash ./scripts/ci/benchmarks-report.sh
+    # Set git config
+    - rm -rf .git/config
+    - git config --global user.email "${GITHUB_EMAIL}"
+    - git config --global user.name "${GITHUB_USER}"
+    - git config remote.origin.url "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/paritytech/${CI_PROJECT_NAME}.git"
+    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
+    # push results to github
+    - git fetch origin gh-pages
+    - git checkout gh-pages
+    - mkdir -p benchmarks/criterion
+    - mkdir -p benchmarks/wasmtime
+    - cp target/ci/output-criterion.txt benchmarks/criterion/
+    - cp target/ci/output-wasmtime.txt benchmarks/wasmtime/
+    - git add benchmarks/
+    - git commit -m "[benchmarks] add benchmarks results for the latest master"
+    - git push origin gh-pages

scripts/ci/sync_wait.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Script waits until file benchmarks/criterion/output-criterion.txt
+# appears
+
+for i in $(seq 1 600); do
+  git pull -q origin gh-pages
+  sleep 1
+  ls benchmarks/criterion/output-criterion.txt 2>/dev/null
+  if [ $? = 0 ]; then
+    break;
+  fi
+  echo "Waiting..."
+done