regression tests & benchmark

.github/workflows/ci.yaml

@@ -21,11 +21,11 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - {mode: stable,  os: ubuntu-latest,  payload: noslow-example }
-          - {mode: stable,  os: macOS-latest,   payload: noslow         }
-          - {mode: stable,  os: windows-latest, payload: noslow         }
-          - {mode: stable,  os: ubuntu-latest,  payload: noslow-mpi     }
-          - {mode: nightly, os: ubuntu-latest,  payload: noslow         }
+          - {mode: stable,  os: ubuntu-latest,  payload: example-noslow-noregression }
+          - {mode: stable,  os: macOS-latest,   payload: noslow-noregression         }
+          - {mode: stable,  os: windows-latest, payload: noslow-noregression         }
+          - {mode: stable,  os: ubuntu-latest,  payload: mpi-noslow-noregression     }
+          - {mode: nightly, os: ubuntu-latest,  payload: noslow-noregression         }
     env:
       GKS_ENCODING: utf8
       GKSwstype: 100       # Needed for Plots-related tests

.github/workflows/regression.yaml

@@ -0,0 +1,74 @@
+name: Regression tests
+on:
+  push:
+    branches:
+      - master
+    tags: ['*']
+  pull_request:
+  schedule:
+    - cron:  '0 4 * * 6'  # Run every Saturday
+concurrency:
+  # Skip intermediate builds: always.
+  # Cancel intermediate builds: only if it is a pull request build.
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
+
+jobs:
+  Benchmark:
+    runs-on: ubuntu-latest
+    steps:
+      # Remove older benchmark comment
+      - name: pr-deleter
+        uses: maheshrayas/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          org: <orgname>
+          repo: <repo>
+          user: github-actions[bot]
+          issue: ${{github.event.number}}
+      - uses: actions/checkout@v2
+      - uses: julia-actions/setup-julia@latest
+        with:
+          version: 1
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: Install dependencies
+        run: |
+          julia --project=benchmark -e '
+          using Pkg
+          Pkg.develop(PackageSpec(; path=pwd()))
+          Pkg.instantiate()'
+      - name: Run benchmarks against master
+        # Remove baseline once merged. Regression tests will only work after this is merged
+        # in master.
+        run: |
+          julia --project=benchmark -e '
+            using BenchmarkCI
+            BenchmarkCI.judge(; baseline="HEAD", retune=true)'
+        if: ${{ github.event_name == 'pull_request' }}
+      - name: Run benchmarks against last release
+        run: |
+          julia --project=benchmark -e '
+            import Pkg
+            baseline="v"*Pkg.TOML.parsefile("Project.toml")["version"]
+            using BenchmarkCI
+            BenchmarkCI.judge(; baseline, retune=true)'
+        if: ${{ github.event_name == 'schedule' ||
+                github.event.push.ref == 'refs/heads/master' }}
+      - name: Print judgement
+        run: |
+          julia --project=benchmark -e '
+            using BenchmarkCI
+            BenchmarkCI.displayjudgement()'
+      - name: Post results
+        run: |
+          julia --project=benchmark -e '
+            using BenchmarkCI
+            BenchmarkCI.postjudge()'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Is report successful
+        run: |
+          res=$(julia --project=benchmark -e '
+                  using BenchmarkCI
+                  BenchmarkCI.displayjudgement()' | grep --count ':x:')
+          if [[ $res -gt 1 ]]; then exit 1; fi

.gitignore

@@ -10,3 +10,5 @@ Manifest.toml
 /LocalPreferences.toml
 .vscode
 .CondaPkg
+/.benchmarkci
+/benchmark/**/*.json

Project.toml

@@ -128,6 +128,7 @@ ASEconvert = "3da9722f-58c2-4165-81be-b4d7253e8fd2"
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
 AtomsIO = "1692102d-eeb4-4df9-807b-c9517f998d44"
 AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"
 ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66"
@@ -150,4 +151,4 @@ WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192"
 wannier90_jll = "c5400fa0-8d08-52c2-913f-1e3f656c1ce9"
 
 [targets]
-test = ["Test", "TestItemRunner", "ASEconvert", "Aqua", "AtomsIO", "AtomsIOPython", "CUDA", "CUDA_Runtime_jll", "ComponentArrays", "DoubleFloats", "FiniteDiff", "FiniteDifferences", "GenericLinearAlgebra", "IntervalArithmetic", "JLD2", "JSON3", "Logging", "Plots", "QuadGK", "Random", "KrylovKit", "Wannier", "WriteVTK", "wannier90_jll"]
+test = ["Test", "TestItemRunner", "ASEconvert", "Aqua", "AtomsIO", "AtomsIOPython", "BenchmarkTools", "CUDA", "CUDA_Runtime_jll", "ComponentArrays", "DoubleFloats", "FiniteDiff", "FiniteDifferences", "GenericLinearAlgebra", "IntervalArithmetic", "JLD2", "JSON3", "Logging", "Plots", "QuadGK", "Random", "KrylovKit", "Wannier", "WriteVTK", "wannier90_jll"]

benchmark/Project.toml

@@ -0,0 +1,10 @@
+[deps]
+AtomsBase = "a963bdd2-2df7-4f54-a1ee-49d51e6be12a"
+BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
+UnitfulAtomic = "a7773ee8-282e-5fa2-be4e-bd808c38a91a"

benchmark/benchmarks.jl

@@ -0,0 +1,8 @@
+using BenchmarkTools
+using TestItemRunner
+
+const SUITE = BenchmarkGroup()
+
+julia_cmd = unsafe_string(Base.JLOptions().julia_bin)
+SUITE["load"] = @benchmarkable run(`$julia_cmd --startup-file=no --project=$(Base.active_project()) -e 'using DFTK'`)
+@run_package_tests filter=ti->(:regression ∈ ti.tags)

benchmark/humongous/Project.toml

@@ -0,0 +1,10 @@
+[deps]
+AtomsIOPython = "9e4c859b-2281-48ef-8059-f50fe53c37b0"
+BenchmarkCI = "20533458-34a3-403d-a444-e18f38190b5b"
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+DFTK = "acf6eb54-70d9-11e9-0013-234b7a5f5337"
+LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MKL = "33e6dc65-8f57-5167-99aa-e5a354878fb2"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"

benchmark/humongous/benchmarks.jl

@@ -0,0 +1,8 @@
+using BenchmarkTools
+using TestItemRunner
+
+const SUITE = BenchmarkGroup()
+
+SUITE["AlSiO2H"] = BenchmarkGroup()
+SUITE["AlSiO2H"] = @benchmarkable @run_package_tests filter=(i->occursin("AlSiO2H.jl", i.filename) &&
+                                                             :debug ∈ i.tags)

benchmark/humongous/run.jl

@@ -0,0 +1,23 @@
+ROOTPATH  = joinpath(@__DIR__, "../..")
+import Pkg
+Pkg.activate(@__DIR__)
+if !isfile(joinpath(@__DIR__, "Manifest.toml"))
+    Pkg.develop(Pkg.PackageSpec(; path=ROOTPATH))
+    Pkg.instantiate()
+end
+
+using BenchmarkCI
+
+import LibGit2
+repo = mktempdir(mktempdir())  # TestItemRunner needs access to parent directory as well.
+LibGit2.clone("https://github.com/epolack/DFTK-testproblems", repo)
+
+# Workaround to be able to benchmark releases before the use of PkgBenchmark.
+script = tempname(repo) * ".jl"
+benchpath = joinpath(ROOTPATH, "benchmark", "humongous", "benchmarks.jl")
+project = dirname(benchpath)
+cp(benchpath, script)
+
+BenchmarkCI.judge(; retune=true, script, project)
+
+BenchmarkCI.displayjudgement()

benchmark/regression/testcases.jl

@@ -0,0 +1,156 @@
+@testsetup module Regression
+using DFTK
+using Unitful
+using UnitfulAtomic
+using AtomsBase
+using ..TestCases: magnesium
+
+high_symmetry = let
+    a = 4.474
+    lattice = [[0, a, a], [a, 0, a], [a, a, 0]]u"bohr"
+    x = 6.711
+    y = 2.237
+    atoms = [
+        Atom(:Cu, [0, 0, 0]u"bohr", magnetic_moment=0),
+        Atom(:O,  [x, y, x]u"bohr", magnetic_moment=0),
+        Atom(:O,  [x, y, y]u"bohr", magnetic_moment=0),
+    ]
+    system = periodic_system(atoms, lattice)
+    merge(DFTK.parse_system(system), (; temperature=0.03, Ecut=10, kgrid=[4,4,4],
+          n_electrons=45))
+end
+high_kpoints = merge(magnesium, (; kgrid=[13,13,13], Ecut=10))
+high_Ecut = merge(magnesium, (; kgrid=[4,4,4], Ecut=60))
+testcases = (; high_symmetry, high_kpoints, high_Ecut)
+end
+
+@testitem "Hamiltonian application" tags=[:regression] setup=[TestCases, Regression] begin
+    using DFTK
+    using LinearAlgebra
+    using BenchmarkTools
+    using .Main: SUITE
+
+    for testcase in Regression.testcases
+        model = Model(testcase.lattice, testcase.atoms, testcase.positions;
+                      testcase.temperature, terms=[Kinetic()])
+        basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
+
+        n_electrons = testcase.n_electrons
+        n_bands = div(n_electrons, 2, RoundUp)
+        ψ = [Matrix(qr(randn(ComplexF64, length(G_vectors(basis, kpt)), n_bands)).Q)
+             for kpt in basis.kpoints]
+        filled_occ = DFTK.filled_occupation(model)
+        occupation = [filled_occ * rand(n_bands) for _ = 1:length(basis.kpoints)]
+        occ_scaling = n_electrons / sum(sum(occupation))
+        occupation = [occ * occ_scaling for occ in occupation]
+
+        (; ham) = energy_hamiltonian(basis, ψ, occupation)
+
+        SUITE["ham"] = BenchmarkGroup()
+        SUITE["ham"] = @benchmarkable for ik = 1:length($(basis.kpoints))
+            $(ham.blocks)[ik]*$ψ[ik]
+        end
+    end
+end
+
+@testitem "Single SCF step" tags=[:regression] setup=[TestCases, Regression] begin
+    using DFTK
+    using BenchmarkTools
+    using .Main: SUITE
+
+    for testcase in Regression.testcases
+        model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
+                          testcase.temperature)
+        basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
+        SUITE["scf"] = BenchmarkGroup()
+        SUITE["scf"] = @benchmarkable self_consistent_field($basis; tol=1e5)
+    end
+end
+
+@testitem "Density + symmetrization" tags=[:regression] setup=[TestCases, Regression] begin
+    using DFTK
+    using BenchmarkTools
+    using .Main: SUITE
+
+    for testcase in Regression.testcases
+        model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
+                          testcase.temperature)
+        basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
+        scfres = self_consistent_field(basis; tol=10)
+
+        ψ, occupation = DFTK.select_occupied_orbitals(basis, scfres.ψ, scfres.occupation;
+                                                      threshold=1e-6)
+
+        SUITE["density", "ρ", "sym"] = BenchmarkGroup()
+        SUITE["density", "ρ"] = @benchmarkable compute_density($basis, $ψ, $occupation)
+        SUITE["density", "sym"] = @benchmarkable DFTK.symmetrize_ρ($basis, $(scfres.ρ))
+    end
+end
+
+@testitem "Basis construction" tags=[:regression] setup=[TestCases, Regression] begin
+    using DFTK
+    using BenchmarkTools
+    using .Main: SUITE
+
+    for testcase in Regression.testcases
+        model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
+                          testcase.temperature)
+        SUITE["basis"] = BenchmarkGroup()
+        SUITE["basis"] = @benchmarkable PlaneWaveBasis($model; Ecut=$(testcase.Ecut),
+                                                       kgrid=$(testcase.kgrid))
+    end
+end
+
+@testitem "Sternheimer" tags=[:regression] setup=[TestCases, Regression] begin
+    using DFTK
+    using BenchmarkTools
+    using .Main: SUITE
+
+    for testcase in Regression.testcases
+        model = model_LDA(testcase.lattice, testcase.atoms, testcase.positions;
+                          testcase.temperature)
+        basis = PlaneWaveBasis(model; testcase.Ecut, testcase.kgrid)
+        scfres = self_consistent_field(basis; tol=10)
+
+        rhs = DFTK.compute_projected_gradient(basis, scfres.ψ, scfres.occupation)
+        SUITE["sternheimer"] = BenchmarkGroup()
+        SUITE["sternheimer"] = @benchmarkable DFTK.solve_ΩplusK_split($scfres, $rhs)
+    end
+end
+
+@testitem "Response with AD" tags=[:regression] setup=[TestCases, Regression] begin
+    using DFTK
+    using BenchmarkTools
+    using LinearAlgebra
+    using ForwardDiff
+    using .Main: SUITE
+
+    function make_basis(ε::T; a=10., Ecut=30) where {T}
+        lattice=T(a) * I(3)  # lattice is a cube of ``a`` Bohrs
+        # Helium at the center of the box
+        atoms     = [ElementPsp(:He; psp=load_psp("hgh/lda/He-q2"))]
+        positions = [[1/2, 1/2, 1/2]]
+
+        model = model_DFT(lattice, atoms, positions, [:lda_x, :lda_c_vwn];
+                          extra_terms=[ExternalFromReal(r -> -ε * (r[1] - a/2))],
+                          symmetries=false)
+        PlaneWaveBasis(model; Ecut, kgrid=[1, 1, 1])  # No k-point sampling on isolated system
+    end
+
+    # dipole moment of a given density (assuming the current geometry)
+    function dipole(basis, ρ)
+        @assert isdiag(basis.model.lattice)
+        a  = basis.model.lattice[1, 1]
+        rr = [a * (r[1] - 1/2) for r in r_vectors(basis)]
+        sum(rr .* ρ) * basis.dvol
+    end
+
+    # Function to compute the dipole for a given field strength
+    function compute_dipole(ε; tol=1e-8, kwargs...)
+        scfres = self_consistent_field(make_basis(ε; kwargs...); tol)
+        dipole(scfres.basis, scfres.ρ)
+    end
+
+    SUITE["response", "ad"] = BenchmarkGroup()
+    SUITE["response", "ad"] = @benchmarkable ForwardDiff.derivative($compute_dipole, 0.0)
+end

benchmark/run.jl

@@ -0,0 +1,14 @@
+ROOTPATH  = joinpath(@__DIR__, "..")
+import Pkg
+Pkg.activate(@__DIR__)
+if !isfile(joinpath(@__DIR__, "Manifest.toml"))
+    Pkg.develop(Pkg.PackageSpec(; path=ROOTPATH))
+    Pkg.instantiate()
+end
+
+using BenchmarkCI
+
+# Remove target once merged. Regression tests will only work after this is merged in master.
+BenchmarkCI.judge(; baseline="HEAD")
+
+BenchmarkCI.displayjudgement()