external humongous run

benchmark/humongous/Project.toml

@@ -1,5 +1,5 @@
 [deps]
-AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
+AtomsIO = "1692102d-eeb4-4df9-807b-c9517f998d44"
 BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"

benchmark/humongous/benchmarks.jl

@@ -1,8 +1,23 @@
 using BenchmarkTools
 using TestItemRunner
 
-const SUITE = BenchmarkGroup()
+function run_scenario(scenario, complexity)
+    scenario_filter(i) = occursin(string(scenario), i.filename) && complexity ∈ i.tags
+    @run_package_tests filter=scenario_filter
+end
 
-SUITE["AlSiO2H"] = BenchmarkGroup()
-SUITE["AlSiO2H"] = @benchmarkable @run_package_tests filter=(i->occursin("AlSiO2H.jl", i.filename) &&
-                                                             :debug ∈ i.tags)
+all_scenarios() = [:AlSiO2H, :Cr19, :Fe2MnAl, :Mn2RuGa, :WFe]
+function make_suite(; scenarios=all_scenarios(), complexity=:debug)
+    @assert complexity ∈ [:debug, :small, :full]
+    @assert all(scenarios .∈ Ref(all_scenarios()))
+
+    suite = BenchmarkGroup()
+
+    for scenario in scenarios
+        suite[scenario] = BenchmarkGroup()
+        suite[scenario] = @benchmarkable run_scenario($scenario, $complexity)
+    end
+    suite
+end
+
+const SUITE = make_suite(; scenarios=[:AlSiO2H])

benchmark/humongous/run.jl

@@ -1,23 +1,43 @@
-ROOTPATH  = joinpath(@__DIR__, "../..")
+ROOTPATH = abspath(joinpath(@__DIR__, "../.."))
 import Pkg
 Pkg.activate(@__DIR__)
 if !isfile(joinpath(@__DIR__, "Manifest.toml"))
     Pkg.develop(Pkg.PackageSpec(; path=ROOTPATH))
     Pkg.instantiate()
 end
 
-using BenchmarkCI
-
+import BenchmarkCI
 import LibGit2
-repo = mktempdir(mktempdir())  # TestItemRunner needs access to parent directory as well.
-LibGit2.clone("https://github.com/epolack/DFTK-testproblems", repo)
 
-# Workaround to be able to benchmark releases before the use of PkgBenchmark.
-script = tempname(repo) * ".jl"
-benchpath = joinpath(ROOTPATH, "benchmark", "humongous", "benchmarks.jl")
-project = dirname(benchpath)
-cp(benchpath, script)
+"""
+Launch with
+```julia
+julia --project=benchmark/humongous -e '
+   include("benchmark/humongous/run.jl")
+   run_benchmark()'
+```
+"""
+function run_benchmark(; retune=false, baseline="origin/master", target="HEAD",
+                       script=nothing)
+    mktempdir(mktempdir()) do repo_dir  # TestItemRunner needs access to parent directory as well.
+        project = joinpath(ROOTPATH, "benchmark", "humongous")
+        # Workaround to be able to benchmark releases before the use of PkgBenchmark.
+        # WARN: In this case, we need PkgBenchmark to be installed globally.
+        if isnothing(script)
+            # We run the default benchmark.
+            script = joinpath(project, "benchmarks.jl")
+        else
+            occursin(ROOTPATH, abspath(script)) &&
+                error("Script should be outside the repository.")
+        end
+        script_copy = joinpath(repo_dir, "benchmarks.jl")
+        cp(script, script_copy)
+
+        LibGit2.clone("https://github.com/epolack/DFTK-testproblems",
+                      joinpath(repo_dir, "test"))
 
-BenchmarkCI.judge(; retune=true, script, project)
+        BenchmarkCI.judge(; baseline, target, retune, script=script_copy, project)
 
-BenchmarkCI.displayjudgement()
+        BenchmarkCI.displayjudgement()
+    end
+end