From 88e8d7af02db3b026677405840f669abc10ae2a2 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Wed, 21 Oct 2020 22:14:21 +0200
Subject: [PATCH 01/43] add third case, only recording

---
 bench/02-basic.R | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 0a2fdca74..a761c4633 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -6,8 +6,6 @@ library(magrittr)
 path <- "sources/here"
 dir.create("plots")
 cache_clear(ask = FALSE)
-cache_activate()
-cache_info()
 
 marker <- purrr::partial(
   bench::mark,
@@ -17,12 +15,30 @@ marker <- purrr::partial(
   memory = TRUE # skip uncached first round
 )
 
+# basically applying cache only
+# No transformer is ran ever because all code is already compliant.
+cache_activate()
+cache_info()
+with_cache <- marker(
+  cache_appyling = {
+    style_pkg(path, filetype = c("R", "rmd"))
+  }
+)
+
+# basically recording cache only
+# transformers are always ran on expressions that were not compliant with the
+# style guide (some but not all expressions are)
 with_cache <- marker(
-  with_cache = {
+  cache_recording = {
+    gert::git_reset_hard(repo = path)
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )
+
 cache_info()
+
+# cache turned off
+# recording and applying, transformers always ran on all expressions.
 gert::git_reset_hard(repo = path)
 cache_deactivate()
 
@@ -31,6 +47,8 @@ without_cache <- marker(
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )
+
+# visualize results
 latest_bm <- bench::cb_read()$benchmarks[[1]]
 split(latest_bm, latest_bm$name) %>%
   purrr::imap(plot_against_base)

From 7f360f1f2bbf764c87c9435268740d218a8a0ed8 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Wed, 21 Oct 2020 23:05:46 +0200
Subject: [PATCH 02/43] simulate reset

---
 bench/02-basic.R | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index a761c4633..7da987375 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -30,7 +30,12 @@ with_cache <- marker(
 # style guide (some but not all expressions are)
 with_cache <- marker(
   cache_recording = {
-    gert::git_reset_hard(repo = path)
+    cat(
+      bench::system_time(gert::git_reset_hard(repo = path))[["process"]],
+      sep = "\n",
+      file = "timing-reset",
+      append = TRUE
+    )
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )
@@ -41,9 +46,19 @@ cache_info()
 # recording and applying, transformers always ran on all expressions.
 gert::git_reset_hard(repo = path)
 cache_deactivate()
-
+time_for_git_reset <- as.numeric(readLines("timing-reset"))
+cat(
+  "Waiting ",
+  round(mean(time_for_git_reset), 3),
+  " seconds on average to simulate git reset. That way, `without_cache` and ",
+  "`cache_recording` are comparable. The 95% interval for reset are (",
+  round(quantile(time_for_git_reset, 0.025), 3), ", ",
+  round(quantile(time_for_git_reset, 0.975), 3), ").",
+  sep = ""
+)
 without_cache <- marker(
   without_cache = {
+    Sys.sleep(mean(time_for_git_reset))
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )

From a07909464b972455556b1ea12234fa97e3a97c62 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Thu, 22 Oct 2020 10:51:29 +0200
Subject: [PATCH 03/43] drop first reset, as first run is not recorded

---
 bench/02-basic.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 7da987375..743821d56 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -46,7 +46,7 @@ cache_info()
 # recording and applying, transformers always ran on all expressions.
 gert::git_reset_hard(repo = path)
 cache_deactivate()
-time_for_git_reset <- as.numeric(readLines("timing-reset"))
+time_for_git_reset <- as.numeric(readLines("timing-reset"))[-1]
 cat(
   "Waiting ",
   round(mean(time_for_git_reset), 3),

From 310e2cb92a5efb03aab90f4d6c928268138204c2 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Thu, 22 Oct 2020 11:14:06 +0200
Subject: [PATCH 04/43] force more idle time

---
 bench/02-basic.R | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 743821d56..36b780b33 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -7,6 +7,7 @@ path <- "sources/here"
 dir.create("plots")
 cache_clear(ask = FALSE)
 
+
 marker <- purrr::partial(
   bench::mark,
   min_iterations = 20,
@@ -19,6 +20,7 @@ marker <- purrr::partial(
 # No transformer is ran ever because all code is already compliant.
 cache_activate()
 cache_info()
+Sys.sleep(3) # make system more idle
 with_cache <- marker(
   cache_appyling = {
     style_pkg(path, filetype = c("R", "rmd"))
@@ -28,6 +30,7 @@ with_cache <- marker(
 # basically recording cache only
 # transformers are always ran on expressions that were not compliant with the
 # style guide (some but not all expressions are)
+Sys.sleep(3) # make system more idle
 with_cache <- marker(
   cache_recording = {
     cat(
@@ -56,6 +59,7 @@ cat(
   round(quantile(time_for_git_reset, 0.975), 3), ").",
   sep = ""
 )
+Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
     Sys.sleep(mean(time_for_git_reset))

From 47e57a7457f760fe8f524ce72e8a55dd3ceb5bf0 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Thu, 22 Oct 2020 11:14:56 +0200
Subject: [PATCH 05/43] more iterations

---
 bench/02-basic.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 36b780b33..519cc89e9 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -10,7 +10,7 @@ cache_clear(ask = FALSE)
 
 marker <- purrr::partial(
   bench::mark,
-  min_iterations = 20,
+  min_iterations = 25,
   check = FALSE,
   filter_gc = FALSE,
   memory = TRUE # skip uncached first round

From 799e9fea65f9e13cc3429ab6b8f05f986df46738 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Fri, 23 Oct 2020 20:21:45 +0200
Subject: [PATCH 06/43] get rid of the 0.012 seconds a reset takes on average.

---
 bench/02-basic.R | 1 -
 1 file changed, 1 deletion(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 519cc89e9..61f7dafc5 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -62,7 +62,6 @@ cat(
 Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
-    Sys.sleep(mean(time_for_git_reset))
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )

From 07d3d67656d8f3e40992588244f3f58c1620f22f Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Fri, 23 Oct 2020 20:48:31 +0200
Subject: [PATCH 07/43] Revert "get rid of the 0.012 seconds a reset takes on
 average."

This reverts commit 799e9fea65f9e13cc3429ab6b8f05f986df46738.
---
 bench/02-basic.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 61f7dafc5..519cc89e9 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -62,6 +62,7 @@ cat(
 Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
+    Sys.sleep(mean(time_for_git_reset))
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )

From 11b1eabd9576d8482daf3d1c9f6d127846ac12b1 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Fri, 23 Oct 2020 20:49:14 +0200
Subject: [PATCH 08/43] pre-compute mean

---
 bench/02-basic.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 519cc89e9..ccbe09e02 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -59,10 +59,11 @@ cat(
   round(quantile(time_for_git_reset, 0.975), 3), ").",
   sep = ""
 )
+mean_time_for_git_reset <- mean(time_for_git_reset)
 Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
-    Sys.sleep(mean(time_for_git_reset))
+    Sys.sleep(mean_time_for_git_reset)
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )

From a878874b81699f8d9802eaf33f8bdcf4b29d2e27 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Fri, 23 Oct 2020 21:02:47 +0200
Subject: [PATCH 09/43] Revert "pre-compute mean"

This reverts commit 11b1eabd9576d8482daf3d1c9f6d127846ac12b1.
---
 bench/02-basic.R | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index ccbe09e02..519cc89e9 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -59,11 +59,10 @@ cat(
   round(quantile(time_for_git_reset, 0.975), 3), ").",
   sep = ""
 )
-mean_time_for_git_reset <- mean(time_for_git_reset)
 Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
-    Sys.sleep(mean_time_for_git_reset)
+    Sys.sleep(mean(time_for_git_reset))
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )

From f87eb0e4b335f8b4f2f034b12dc66f5ee79e06be Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Fri, 23 Oct 2020 21:10:21 +0200
Subject: [PATCH 10/43] Revert "Revert "pre-compute mean""

This reverts commit a878874b81699f8d9802eaf33f8bdcf4b29d2e27.
---
 bench/02-basic.R | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 519cc89e9..ccbe09e02 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -59,10 +59,11 @@ cat(
   round(quantile(time_for_git_reset, 0.975), 3), ").",
   sep = ""
 )
+mean_time_for_git_reset <- mean(time_for_git_reset)
 Sys.sleep(3) # make system more idle
 without_cache <- marker(
   without_cache = {
-    Sys.sleep(mean(time_for_git_reset))
+    Sys.sleep(mean_time_for_git_reset)
     style_pkg(path, filetype = c("R", "rmd"))
   }
 )

From d947c425c7da1fe2ab9cd850baa43215fc79dbbd Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 01:11:40 +0200
Subject: [PATCH 11/43] lock R and package versions for more stability

Avoid that base branch cb was built with different package versions.
---
 .github/workflows/benchmarking.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 73f2b45b8..861e17474 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -21,6 +21,11 @@ jobs:
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch
       - name: Setup R
         uses: r-lib/actions/setup-r@master
+        with:
+          r: '4.0.0'
+          rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/291"
+          http-user-agent: "R/4.0.0 (ubuntu-16.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" }
+
       - name: Query dependencies
         run: |
           install.packages('remotes')

From 94770cd36442e6d3f46fa3630536dddbed8e2b20 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 01:13:49 +0200
Subject: [PATCH 12/43] switch to ubuntu from macOS

---
 .github/workflows/benchmarking.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 861e17474..eb73b8723 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -10,7 +10,7 @@ name: Continuous Benchmarks
 
 jobs:
   build:
-    runs-on: macOS-latest
+    runs-on: ubuntu-18.04
     steps:
       - name: Checkout repo
         with:
@@ -23,8 +23,8 @@ jobs:
         uses: r-lib/actions/setup-r@master
         with:
           r: '4.0.0'
-          rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/291"
-          http-user-agent: "R/4.0.0 (ubuntu-16.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" }
+          rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
+          http-user-agent: "R/4.0.0 (ubuntu-18.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" }
 
       - name: Query dependencies
         run: |

From 8e535d694866a69a3bee57651249d1b99f6115fb Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 01:15:36 +0200
Subject: [PATCH 13/43] parsing error

---
 .github/workflows/benchmarking.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index eb73b8723..d84583d41 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -24,7 +24,7 @@ jobs:
         with:
           r: '4.0.0'
           rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
-          http-user-agent: "R/4.0.0 (ubuntu-18.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions" }
+          http-user-agent: "R/4.0.0 (ubuntu-18.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions"
 
       - name: Query dependencies
         run: |

From 47ffeaba6b38c74810d3467607b07440ef1a0b58 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 01:26:31 +0200
Subject: [PATCH 14/43] install system dependencies

---
 .github/workflows/benchmarking.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index d84583d41..5ea7abdfb 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -40,7 +40,14 @@ jobs:
           path: ${{ env.R_LIBS_USER }}
           key: ${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
           restore-keys: ${{ hashFiles('.github/R-version') }}-1-
-
+      - name: Install system dependencies
+        if: runner.os == 'Linux'
+        env:
+          RHUB_PLATFORM: linux-x86_64-ubuntu-gcc
+        run: |
+          Rscript -e "remotes::install_github('r-hub/sysreqs')"
+          sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
+          sudo -s eval "$sysreqs"
       - name: Install dependencies
         run: |
           Rscript -e "install.packages(c('gert', 'ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github('r-lib/bench')"

From 56fa1cd0479bcdb5e3ed715f48621d735b56ebb1 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 01:32:46 +0200
Subject: [PATCH 15/43] don't cache deps

---
 .github/workflows/benchmarking.yaml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 5ea7abdfb..6ea07ae85 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -32,14 +32,6 @@ jobs:
           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
         shell: Rscript {0}
-
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v1
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ hashFiles('.github/R-version') }}-1-
       - name: Install system dependencies
         if: runner.os == 'Linux'
         env:

From 7a37d71a79b452ac5c42b9b79ab621adb456e075 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 08:19:20 +0100
Subject: [PATCH 16/43] setup git user

---
 .github/workflows/benchmarking.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 6ea07ae85..d2c33084a 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -16,6 +16,10 @@ jobs:
         with:
           fetch-depth: 0
         uses: actions/checkout@master
+      - name: Set up git user
+	      run: |
+	        git config --local user.email "actions@github.com"
+	        git config --local user.name "GitHub Actions"
       - name: Ensure base branch is fetched
         if: ${{ github.event_name == 'pull_request' }}
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch

From 391f5d57fcf856f993ef5e45c91e74ad977f6b27 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 08:50:38 +0100
Subject: [PATCH 17/43] set up git user again

---
 .github/workflows/benchmarking.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index d2c33084a..4daa559eb 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -18,8 +18,8 @@ jobs:
         uses: actions/checkout@master
       - name: Set up git user
 	      run: |
-	        git config --local user.email "actions@github.com"
 	        git config --local user.name "GitHub Actions"
+	        git config --local user.email "actions@github.com"
       - name: Ensure base branch is fetched
         if: ${{ github.event_name == 'pull_request' }}
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch

From ba698053f90d6c77b5763ba319e3e17728f25fbc Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 08:58:00 +0100
Subject: [PATCH 18/43] drop all but benchmarks runs

---
 .github/workflows/R-CMD-check.yaml | 98 ------------------------------
 1 file changed, 98 deletions(-)
 delete mode 100644 .github/workflows/R-CMD-check.yaml

diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
deleted file mode 100644
index 0f5ea8220..000000000
--- a/.github/workflows/R-CMD-check.yaml
+++ /dev/null
@@ -1,98 +0,0 @@
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-name: R-CMD-check
-
-jobs:
-  R-CMD-check:
-    runs-on: ${{ matrix.config.os }}
-
-    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
-
-    strategy:
-      fail-fast: false
-      matrix:
-        config:
-          - {os: macOS-latest,   r: 'devel'}
-          - {os: macOS-latest,   r: 'release'}
-          - {os: windows-latest, r: 'release'}
-          - {os: windows-latest, r: '3.6'}
-          - {os: ubuntu-16.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          - {os: ubuntu-16.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          - {os: ubuntu-16.04,   r: '3.5',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          - {os: ubuntu-16.04,   r: '3.4',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-          - {os: ubuntu-16.04,   r: '3.3',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
-
-    env:
-      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
-      RSPM: ${{ matrix.config.rspm }}
-      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
-      _R_CHECK_FORCE_SUGGESTS_: false
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: r-lib/actions/setup-r@master
-        with:
-          r-version: ${{ matrix.config.r }}
-
-      - uses: r-lib/actions/setup-pandoc@master
-
-      - name: Query dependencies
-        run: |
-          install.packages('remotes')
-          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
-          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
-        shell: Rscript {0}
-
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v1
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
-
-      - name: Install system dependencies
-        if: runner.os == 'Linux'
-        env:
-          RHUB_PLATFORM: linux-x86_64-ubuntu-gcc
-        run: |
-          Rscript -e "remotes::install_github('r-hub/sysreqs')"
-          sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
-          sudo -s eval "$sysreqs"
-
-      - name: Install dependencies
-        run: |
-          remotes::install_deps(dependencies = TRUE)
-          remotes::install_cran("rcmdcheck")
-        shell: Rscript {0}
-
-      - name: Session info
-        run: |
-          options(width = 100)
-          pkgs <- installed.packages()[, "Package"]
-          sessioninfo::session_info(pkgs, include_base = TRUE)
-        shell: Rscript {0}
-
-      - name: Check
-        env:
-          _R_CHECK_CRAN_INCOMING_: false
-        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
-        shell: Rscript {0}
-
-      - name: Show testthat output
-        if: always()
-        run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true
-        shell: bash
-
-      - name: Upload check results
-        if: failure()
-        uses: actions/upload-artifact@master
-        with:
-          name: ${{ runner.os }}-r${{ matrix.config.r }}-results
-          path: check

From 3f3b6121b40111765829a6d35775d43e87225970 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 08:59:59 +0100
Subject: [PATCH 19/43] fix indent

---
 .github/workflows/benchmarking.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 4daa559eb..5364e8b54 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -17,9 +17,9 @@ jobs:
           fetch-depth: 0
         uses: actions/checkout@master
       - name: Set up git user
-	      run: |
-	        git config --local user.name "GitHub Actions"
-	        git config --local user.email "actions@github.com"
+        run: |
+          git config --local user.name "GitHub Actions"
+          git config --local user.email "actions@github.com"
       - name: Ensure base branch is fetched
         if: ${{ github.event_name == 'pull_request' }}
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch

From d2a203420d200baf95d90a1a9904bb56b12e88b6 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:11:52 +0100
Subject: [PATCH 20/43] install more system deps

---
 .github/workflows/benchmarking.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 5364e8b54..5b278c84d 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -44,6 +44,7 @@ jobs:
           Rscript -e "remotes::install_github('r-hub/sysreqs')"
           sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
           sudo -s eval "$sysreqs"
+          apt-get install libcurl4-openssl-dev
       - name: Install dependencies
         run: |
           Rscript -e "install.packages(c('gert', 'ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github('r-lib/bench')"

From c30eb623759228586414d866a1043dd24bd90bfe Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:15:28 +0100
Subject: [PATCH 21/43] sudo apt install

---
 .github/workflows/benchmarking.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 5b278c84d..7f48209db 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -44,7 +44,7 @@ jobs:
           Rscript -e "remotes::install_github('r-hub/sysreqs')"
           sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
           sudo -s eval "$sysreqs"
-          apt-get install libcurl4-openssl-dev
+          sudo apt-get install libcurl4-openssl-dev
       - name: Install dependencies
         run: |
           Rscript -e "install.packages(c('gert', 'ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github('r-lib/bench')"

From 4a9416647bbb8804e4e43bbb54168d110aeb8313 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:19:06 +0100
Subject: [PATCH 22/43] actually use RStudio Package Manager

---
 .github/workflows/benchmarking.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 7f48209db..cb6a9068a 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -26,9 +26,10 @@ jobs:
       - name: Setup R
         uses: r-lib/actions/setup-r@master
         with:
-          r: '4.0.0'
-          rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
+          r-version: '4.0.0'
           http-user-agent: "R/4.0.0 (ubuntu-18.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions"
+        env:
+          RSPM: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
 
       - name: Query dependencies
         run: |

From c7c7dfd778abd47ec4d3417eb34d89e91a781789 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:24:37 +0100
Subject: [PATCH 23/43] set RStudio package manager globally

---
 .github/workflows/benchmarking.yaml | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index cb6a9068a..cad8ceab9 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -1,3 +1,4 @@
+name: Continuous Benchmarks
 on:
   push:
     branches:
@@ -5,9 +6,8 @@ on:
   pull_request:
     branches:
     - master
-
-name: Continuous Benchmarks
-
+env:
+  RSPM: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
 jobs:
   build:
     runs-on: ubuntu-18.04
@@ -28,9 +28,6 @@ jobs:
         with:
           r-version: '4.0.0'
           http-user-agent: "R/4.0.0 (ubuntu-18.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions"
-        env:
-          RSPM: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
-
       - name: Query dependencies
         run: |
           install.packages('remotes')

From b60d59504b952d34a9bcc12dc4ad96796d21db67 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:30:02 +0100
Subject: [PATCH 24/43] more close to r cmd check template

---
 .github/workflows/benchmarking.yaml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index cad8ceab9..09c218e68 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -7,10 +7,19 @@ on:
     branches:
     - master
 env:
-  RSPM: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"
+  RSPM:
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {os: ubuntu-18.04,   r: '4.0.0', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"}
+
+    env:
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      RSPM: ${{ matrix.config.rspm }}
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - name: Checkout repo
         with:
@@ -25,9 +34,6 @@ jobs:
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch
       - name: Setup R
         uses: r-lib/actions/setup-r@master
-        with:
-          r-version: '4.0.0'
-          http-user-agent: "R/4.0.0 (ubuntu-18.04) R (4.0.0 x86_64-pc-linux-gnu x86_64 linux-gnu) on GitHub Actions"
       - name: Query dependencies
         run: |
           install.packages('remotes')

From 7f1e9fb5bb6c30e1735a72d60a095a26650a014b Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:32:00 +0100
Subject: [PATCH 25/43] specify runs-on key

---
 .github/workflows/benchmarking.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 09c218e68..a6f3f9b4d 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -10,6 +10,7 @@ env:
   RSPM:
 jobs:
   build:
+    runs-on: ${{ matrix.config.os }}
     strategy:
       fail-fast: false
       matrix:

From 6308df5df7c5226d0ed159f3c078f365ee18bc14 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 09:40:50 +0100
Subject: [PATCH 26/43] specify R version

---
 .github/workflows/benchmarking.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index a6f3f9b4d..2a9eb737d 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -35,12 +35,22 @@ jobs:
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch
       - name: Setup R
         uses: r-lib/actions/setup-r@master
+        with:
+          r-version: ${{ matrix.config.r }}
       - name: Query dependencies
         run: |
           install.packages('remotes')
           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
         shell: Rscript {0}
+      - name: Cache R packages
+        if: runner.os != 'Windows'
+        uses: actions/cache@v1
+        with:
+          path: ${{ env.R_LIBS_USER }}
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+
       - name: Install system dependencies
         if: runner.os == 'Linux'
         env:

From b1b646b29d40ffd16ec9fab64c31556b5cf83576 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 14:37:26 +0100
Subject: [PATCH 27/43] Revert "specify R version"

This reverts commit 6308df5df7c5226d0ed159f3c078f365ee18bc14.
---
 .github/workflows/benchmarking.yaml | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 2a9eb737d..a6f3f9b4d 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -35,22 +35,12 @@ jobs:
         run: git branch $GITHUB_BASE_REF remotes/origin/$GITHUB_BASE_REF; git branch
       - name: Setup R
         uses: r-lib/actions/setup-r@master
-        with:
-          r-version: ${{ matrix.config.r }}
       - name: Query dependencies
         run: |
           install.packages('remotes')
           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
         shell: Rscript {0}
-      - name: Cache R packages
-        if: runner.os != 'Windows'
-        uses: actions/cache@v1
-        with:
-          path: ${{ env.R_LIBS_USER }}
-          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
-          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
-
       - name: Install system dependencies
         if: runner.os == 'Linux'
         env:

From 255917034707a36ebd22e3af621e4fbcb754aaf9 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 14:41:09 +0100
Subject: [PATCH 28/43] use all, not cran

---
 .github/workflows/benchmarking.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index a6f3f9b4d..0e07dd50c 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - {os: ubuntu-18.04,   r: '4.0.0', rspm: "https://packagemanager.rstudio.com/cran/__linux__/bionic/291"}
+          - {os: ubuntu-18.04,   r: '4.0.0', rspm: 'https://packagemanager.rstudio.com/all/__linux__/bionic/291'}
 
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true

From a2f2d2df4677c25a289426f8bb10fa9e599023e2 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 14:56:55 +0100
Subject: [PATCH 29/43] cache deps

---
 .github/workflows/benchmarking.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 0e07dd50c..323c45ff0 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -41,6 +41,13 @@ jobs:
           saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
           writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
         shell: Rscript {0}
+      - name: Cache R packages
+        if: runner.os != 'Windows'
+        uses: actions/cache@v1
+        with:
+          path: ${{ env.R_LIBS_USER }}
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-3-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-3-
       - name: Install system dependencies
         if: runner.os == 'Linux'
         env:

From 320e05eddf5ce9c5f74c46ee96931d66e7f62537 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 15:06:45 +0100
Subject: [PATCH 30/43] more sys deps

---
 .github/workflows/benchmarking.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 323c45ff0..81a09013e 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -56,7 +56,7 @@ jobs:
           Rscript -e "remotes::install_github('r-hub/sysreqs')"
           sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
           sudo -s eval "$sysreqs"
-          sudo apt-get install libcurl4-openssl-dev
+          sudo apt-get install libcurl4-openssl-dev libgit2-dev
       - name: Install dependencies
         run: |
           Rscript -e "install.packages(c('gert', 'ggplot2', 'purrr'))" -e "remotes::install_deps(dependencies = TRUE); remotes::install_github('r-lib/bench')"

From dc9026e0aafffb0fc10214d045e5b9c331a8c869 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 16:56:39 +0100
Subject: [PATCH 31/43] bump to see if cache works


From a7ffcf9d59d927122b5b315c2d39d31cdd93146b Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 17:19:57 +0100
Subject: [PATCH 32/43] comment on PR

---
 .github/workflows/benchmarking.yaml | 8 ++++++++
 bench/01-declarations.R             | 4 ++++
 2 files changed, 12 insertions(+)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 81a09013e..1d1986aa2 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -69,6 +69,8 @@ jobs:
           path: bench/sources/here
       - name: Fetch existing benchmarks
         run: Rscript -e 'rlang::with_handlers(bench::cb_fetch(), error = function(e) paste("Could not fetch benchmarks, skipping. The error was", conditionMessage(e)))'
+      - name: Prepare PR comment
+        run: echo 'Here is how the current PR would change benchmark results:\n' > bench/pr-comment
       - name: Run benchmarks
         run: Rscript -e 'bench::cb_run()'
       - name: Show benchmarks
@@ -80,3 +82,9 @@ jobs:
       - name: Push benchmarks
         if: ${{ github.event_name == 'push' }}
         run: Rscript -e "bench::cb_push()"
+      - name: comment PR
+        uses: machine-learning-apps/pr-comment@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          path: bench/pr-comment
diff --git a/bench/01-declarations.R b/bench/01-declarations.R
index 96fc033a8..6950961ea 100644
--- a/bench/01-declarations.R
+++ b/bench/01-declarations.R
@@ -22,6 +22,10 @@ plot_against_base <- function(new_bm,
         dplyr::filter(.data$name %in% !!name)
       reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
       new_bm <- dplyr::bind_rows(reference, new_bm)
+      stopifnot(nrow(new_bm) == 2)
+      diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
+      pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
+      cat(pr_comment, file = "pr-comment", append = TRUE)
     }
   }
   new_bm$branch <- factor(new_bm$expression)

From bb72225492fec5e3cddd1507895da8faa6fcb86b Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 17:30:25 +0100
Subject: [PATCH 33/43] debug

---
 bench/01-declarations.R | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bench/01-declarations.R b/bench/01-declarations.R
index 6950961ea..695c26b08 100644
--- a/bench/01-declarations.R
+++ b/bench/01-declarations.R
@@ -22,6 +22,7 @@ plot_against_base <- function(new_bm,
         dplyr::filter(.data$name %in% !!name)
       reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
       new_bm <- dplyr::bind_rows(reference, new_bm)
+      print(new_bm)
       stopifnot(nrow(new_bm) == 2)
       diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
       pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")

From 1a59a8d5061a52895fc665b3f39fc0da66bbad0c Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 17:57:43 +0100
Subject: [PATCH 34/43] more debugging

---
 bench/01-declarations.R | 23 +++++++++++++++--------
 bench/02-basic.R        |  2 +-
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/bench/01-declarations.R b/bench/01-declarations.R
index 695c26b08..3a3a58c13 100644
--- a/bench/01-declarations.R
+++ b/bench/01-declarations.R
@@ -17,16 +17,23 @@ plot_against_base <- function(new_bm,
   if (any(commit_is_reference) && Sys.getenv("GITHUB_BASE_REF") != "") {
     # if a pull request
     reference <- bm[commit_is_reference, "benchmarks"][[1]][[1]]
-    if (nrow(reference) > 0 && "name" %in% names(reference)) {
+    if ("name" %in% names(reference)) {
+      # if benchmark exists in base branch
       reference <- reference %>%
         dplyr::filter(.data$name %in% !!name)
-      reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
-      new_bm <- dplyr::bind_rows(reference, new_bm)
-      print(new_bm)
-      stopifnot(nrow(new_bm) == 2)
-      diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
-      pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
-      cat(pr_comment, file = "pr-comment", append = TRUE)
+      if (nrow(reference) > 0) {
+        # if benchmark exists in base branch
+        reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
+        print(new_bm)
+        print(new_bm$name)
+        print(reference)
+        print(reference$name)
+        new_bm <- dplyr::bind_rows(reference, new_bm)
+        stopifnot(nrow(new_bm) == 2)
+        diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
+        pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
+        cat(pr_comment, file = "pr-comment", append = TRUE)
+      }
     }
   }
   new_bm$branch <- factor(new_bm$expression)
diff --git a/bench/02-basic.R b/bench/02-basic.R
index ccbe09e02..29e389154 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -10,7 +10,7 @@ cache_clear(ask = FALSE)
 
 marker <- purrr::partial(
   bench::mark,
-  min_iterations = 25,
+  min_iterations = 5,
   check = FALSE,
   filter_gc = FALSE,
   memory = TRUE # skip uncached first round

From f1e8cf34c0b3b46bf312aa4a50dd6c16943ff142 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 18:09:53 +0100
Subject: [PATCH 35/43] down and upload

---
 .github/workflows/benchmarking.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 1d1986aa2..7016727b5 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -82,9 +82,16 @@ jobs:
       - name: Push benchmarks
         if: ${{ github.event_name == 'push' }}
         run: Rscript -e "bench::cb_push()"
+      - uses: actions/upload-artifact@v1
+        with:
+          name: pr-comment
+          path: bench/pr-comment
+      - uses: actions/download-artifact@v1
+        with:
+          name: pr-comment
       - name: comment PR
         uses: machine-learning-apps/pr-comment@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          path: bench/pr-comment
+          path: pr-comment

From bb9db8350dd587705b8c818d1dca749acfc158f1 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 18:20:04 +0100
Subject: [PATCH 36/43] move closer to the example

---
 .github/workflows/benchmarking.yaml | 6 +++---
 bench/01-declarations.R             | 4 +++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 7016727b5..b19b3bb38 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -84,14 +84,14 @@ jobs:
         run: Rscript -e "bench::cb_push()"
       - uses: actions/upload-artifact@v1
         with:
-          name: pr-comment
+          name: results
           path: bench/pr-comment
       - uses: actions/download-artifact@v1
         with:
-          name: pr-comment
+          name: results
       - name: comment PR
         uses: machine-learning-apps/pr-comment@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          path: pr-comment
+          path: results/info.txt
diff --git a/bench/01-declarations.R b/bench/01-declarations.R
index 3a3a58c13..76d24fe0f 100644
--- a/bench/01-declarations.R
+++ b/bench/01-declarations.R
@@ -32,7 +32,9 @@ plot_against_base <- function(new_bm,
         stopifnot(nrow(new_bm) == 2)
         diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
         pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
-        cat(pr_comment, file = "pr-comment", append = TRUE)
+        file <- "pr-comment/info.txt"
+        dir.create(dirname(file))
+        cat(pr_comment, file = file, append = TRUE)
       }
     }
   }

From 7cb2ff5992dfb1bd9072e08798a8f817d1d6a4a7 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 18:40:48 +0100
Subject: [PATCH 37/43] create directory

---
 .github/workflows/benchmarking.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index b19b3bb38..d6801fa97 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -70,7 +70,9 @@ jobs:
       - name: Fetch existing benchmarks
         run: Rscript -e 'rlang::with_handlers(bench::cb_fetch(), error = function(e) paste("Could not fetch benchmarks, skipping. The error was", conditionMessage(e)))'
       - name: Prepare PR comment
-        run: echo 'Here is how the current PR would change benchmark results:\n' > bench/pr-comment
+        run: |
+          mkdir -p bench/pr-comment
+          echo 'Here is how the current PR would change benchmark results:\n' > bench/pr-comment/info.txt
       - name: Run benchmarks
         run: Rscript -e 'bench::cb_run()'
       - name: Show benchmarks

From c56e2323bf89c6b02f5275ed0bc9c9c218f02023 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 19:15:41 +0100
Subject: [PATCH 38/43] fix file

---
 bench/01-declarations.R | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bench/01-declarations.R b/bench/01-declarations.R
index 76d24fe0f..48f03c028 100644
--- a/bench/01-declarations.R
+++ b/bench/01-declarations.R
@@ -32,9 +32,7 @@ plot_against_base <- function(new_bm,
         stopifnot(nrow(new_bm) == 2)
         diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
         pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
-        file <- "pr-comment/info.txt"
-        dir.create(dirname(file))
-        cat(pr_comment, file = file, append = TRUE)
+        cat(pr_comment, file = "pr-comment/info.txt", append = TRUE)
       }
     }
   }

From 0a1ccfda090931ff951fb36e557f8e818f9edc21 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 20:50:03 +0100
Subject: [PATCH 39/43] remove unused env variable

---
 .github/workflows/benchmarking.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index d6801fa97..4f5740e1a 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -6,8 +6,6 @@ on:
   pull_request:
     branches:
     - master
-env:
-  RSPM:
 jobs:
   build:
     runs-on: ${{ matrix.config.os }}

From aa99a4c378fcee293d2f88e62bf60ef29c60c8af Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 20:55:42 +0100
Subject: [PATCH 40/43] don't comment, only print in log

---
 .github/workflows/benchmarking.yaml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/benchmarking.yaml b/.github/workflows/benchmarking.yaml
index 4f5740e1a..3dedc6473 100644
--- a/.github/workflows/benchmarking.yaml
+++ b/.github/workflows/benchmarking.yaml
@@ -90,8 +90,5 @@ jobs:
         with:
           name: results
       - name: comment PR
-        uses: machine-learning-apps/pr-comment@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          path: results/info.txt
+        #uses: machine-learning-apps/pr-comment@master
+        run: cat bench/pr-comment/info.txt

From 8aefa1eb1a5a1dc83d08420bd150a2283096918b Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 21:01:59 +0100
Subject: [PATCH 41/43] Revert "drop all but benchmarks runs"

This reverts commit ba698053f90d6c77b5763ba319e3e17728f25fbc.
---
 .github/workflows/R-CMD-check.yaml | 98 ++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100644 .github/workflows/R-CMD-check.yaml

diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
new file mode 100644
index 000000000..0f5ea8220
--- /dev/null
+++ b/.github/workflows/R-CMD-check.yaml
@@ -0,0 +1,98 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+name: R-CMD-check
+
+jobs:
+  R-CMD-check:
+    runs-on: ${{ matrix.config.os }}
+
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {os: macOS-latest,   r: 'devel'}
+          - {os: macOS-latest,   r: 'release'}
+          - {os: windows-latest, r: 'release'}
+          - {os: windows-latest, r: '3.6'}
+          - {os: ubuntu-16.04,   r: 'release', rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-16.04,   r: 'oldrel',  rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-16.04,   r: '3.5',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-16.04,   r: '3.4',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+          - {os: ubuntu-16.04,   r: '3.3',     rspm: "https://packagemanager.rstudio.com/cran/__linux__/xenial/latest"}
+
+    env:
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      RSPM: ${{ matrix.config.rspm }}
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      _R_CHECK_FORCE_SUGGESTS_: false
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: r-lib/actions/setup-r@master
+        with:
+          r-version: ${{ matrix.config.r }}
+
+      - uses: r-lib/actions/setup-pandoc@master
+
+      - name: Query dependencies
+        run: |
+          install.packages('remotes')
+          saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2)
+          writeLines(sprintf("R-%i.%i", getRversion()$major, getRversion()$minor), ".github/R-version")
+        shell: Rscript {0}
+
+      - name: Cache R packages
+        if: runner.os != 'Windows'
+        uses: actions/cache@v1
+        with:
+          path: ${{ env.R_LIBS_USER }}
+          key: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-${{ hashFiles('.github/depends.Rds') }}
+          restore-keys: ${{ runner.os }}-${{ hashFiles('.github/R-version') }}-1-
+
+      - name: Install system dependencies
+        if: runner.os == 'Linux'
+        env:
+          RHUB_PLATFORM: linux-x86_64-ubuntu-gcc
+        run: |
+          Rscript -e "remotes::install_github('r-hub/sysreqs')"
+          sysreqs=$(Rscript -e "cat(sysreqs::sysreq_commands('DESCRIPTION'))")
+          sudo -s eval "$sysreqs"
+
+      - name: Install dependencies
+        run: |
+          remotes::install_deps(dependencies = TRUE)
+          remotes::install_cran("rcmdcheck")
+        shell: Rscript {0}
+
+      - name: Session info
+        run: |
+          options(width = 100)
+          pkgs <- installed.packages()[, "Package"]
+          sessioninfo::session_info(pkgs, include_base = TRUE)
+        shell: Rscript {0}
+
+      - name: Check
+        env:
+          _R_CHECK_CRAN_INCOMING_: false
+        run: rcmdcheck::rcmdcheck(args = c("--no-manual", "--as-cran"), error_on = "warning", check_dir = "check")
+        shell: Rscript {0}
+
+      - name: Show testthat output
+        if: always()
+        run: find check -name 'testthat.Rout*' -exec cat '{}' \; || true
+        shell: bash
+
+      - name: Upload check results
+        if: failure()
+        uses: actions/upload-artifact@master
+        with:
+          name: ${{ runner.os }}-r${{ matrix.config.r }}-results
+          path: check

From 2ed7b0e6d444d23af0b552b9e32e5842cbeb4c52 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 21:02:31 +0100
Subject: [PATCH 42/43] scale up again

---
 bench/02-basic.R | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench/02-basic.R b/bench/02-basic.R
index 29e389154..ccbe09e02 100644
--- a/bench/02-basic.R
+++ b/bench/02-basic.R
@@ -10,7 +10,7 @@ cache_clear(ask = FALSE)
 
 marker <- purrr::partial(
   bench::mark,
-  min_iterations = 5,
+  min_iterations = 25,
   check = FALSE,
   filter_gc = FALSE,
   memory = TRUE # skip uncached first round

From 5f34131e635dd8b4b3fe6af044c3e48b86702859 Mon Sep 17 00:00:00 2001
From: Lorenz Walthert <lorenz.walthert@icloud.com>
Date: Sun, 25 Oct 2020 21:18:12 +0100
Subject: [PATCH 43/43] see if we can get rid of extra check

---
 bench/01-declarations.R | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/bench/01-declarations.R b/bench/01-declarations.R
index 48f03c028..be8cdcd2b 100644
--- a/bench/01-declarations.R
+++ b/bench/01-declarations.R
@@ -17,23 +17,17 @@ plot_against_base <- function(new_bm,
   if (any(commit_is_reference) && Sys.getenv("GITHUB_BASE_REF") != "") {
     # if a pull request
     reference <- bm[commit_is_reference, "benchmarks"][[1]][[1]]
-    if ("name" %in% names(reference)) {
+    # if benchmark exists in base branch
+    reference <- reference %>%
+      dplyr::filter(.data$name %in% !!name)
+    if (nrow(reference) > 0) {
       # if benchmark exists in base branch
-      reference <- reference %>%
-        dplyr::filter(.data$name %in% !!name)
-      if (nrow(reference) > 0) {
-        # if benchmark exists in base branch
-        reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
-        print(new_bm)
-        print(new_bm$name)
-        print(reference)
-        print(reference$name)
-        new_bm <- dplyr::bind_rows(reference, new_bm)
-        stopifnot(nrow(new_bm) == 2)
-        diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
-        pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
-        cat(pr_comment, file = "pr-comment/info.txt", append = TRUE)
-      }
+      reference$expression <- bench:::new_bench_expr(Sys.getenv("GITHUB_BASE_REF"))
+      new_bm <- dplyr::bind_rows(reference, new_bm)
+      stopifnot(nrow(new_bm) == 2)
+      diff_in_percent <- round(100 * diff(new_bm$p50) / new_bm$p50[1])
+      pr_comment <- glue::glue("* {name}: {new_bm$p50[1]} -> {new_bm$p50[2]} ({diff_in_percent}%)\n")
+      cat(pr_comment, file = "pr-comment/info.txt", append = TRUE)
     }
   }
   new_bm$branch <- factor(new_bm$expression)