feat: add rust container flavor (#351)

* feat: add rust container flavor * ci: fix linter findings * ci: ignore some linter findings * ci: run for all flavours * ci: fix tests for rust container flavor * chore: add pkg-config * ci: fix linter findings * chore: modify build and push workflow for flavors * chore: reduce container size * chore: add Docker extension to devcontainer.json * chore: update rust from 1.76.0 to 1.77.2 * ci: publish result of all testsuites * ci: fix duplicate artifact * Update ci.yml * ci: different fix for multiple artifacts * ci: update all automation to cope with multiple containers * chore: move update-apt-packages to own action * chore: add flip-link package * chore: process review comments * chore: merge the -vscode image into the main image * Update build-push.yml * chore: minor fix after merge * chore: move Rust container to Ubuntu 24.04 * chore: update Rust to latest version * chore: update Docker cli from 25.0.3 to 26.1.1 * chore: add more test for Rust container * chore: reduce duplication in test code * chore: add test for coverage * chore: remove unused bats tags * chore: don't use run in bats tests when not needed See also: https://bats-core.readthedocs.io/en/stable/writing-tests.html#when-not-to-use-run * chore: add mutation testing support to Rust container * chore: correct test name to beter reflect its actual scope * docs: update documentation
philips-software · May 8, 2024 · f7b357b · f7b357b
1 parent 5804c2b
commit f7b357b
Show file tree

Hide file tree

Showing 68 changed files with 819 additions and 250 deletions.
diff --git a/.devcontainer/Dockerfile → .devcontainer/cpp/Dockerfile b/.devcontainer/Dockerfile → .devcontainer/cpp/Dockerfile
@@ -3,7 +3,7 @@ FROM ubuntu:24.04@sha256:3f85b7caad41a95462cf5b787d8a04604c8262cdcdf9a472b8c52ef
 ARG BATS_VERSION=1.10.0
 ARG CCACHE_VERSION=4.9.1
 ARG CLANG_VERSION=17
-ARG DOCKER_VERSION=25.0.3
+ARG DOCKER_VERSION=26.1.1
 ARG MULL_VERSION=main
 ARG INCLUDE_WHAT_YOU_USE_VERSION=0.21
 ARG XWIN_VERSION=0.5.0
@@ -15,7 +15,7 @@ HEALTHCHECK NONE
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # Install the base system with all tool dependencies
-COPY .devcontainer/apt-requirements-base.json /tmp/apt-requirements-base.json
+COPY .devcontainer/cpp/apt-requirements-base.json /tmp/apt-requirements-base.json
 # hadolint ignore=DL3008
 RUN apt-get update && apt-get install -y --no-install-recommends jq \
  && jq -r 'to_entries | .[] | .key + "=" + .value' /tmp/apt-requirements-base.json | xargs apt-get install -y --no-install-recommends \
@@ -27,7 +27,7 @@ RUN wget -qO /usr/local/share/ca-certificates/Cisco_Umbrella_Root_CA.crt https:/
  && update-ca-certificates
 
 # Install some tools via pip to get more recent versions
-COPY .devcontainer/requirements.txt /tmp/requirements.txt
+COPY .devcontainer/cpp/requirements.txt /tmp/requirements.txt
 RUN python3 -m pip install --break-system-packages --require-hashes --no-cache-dir -r /tmp/requirements.txt \
  && rm -rf /tmp/requirements.txt
 
@@ -36,7 +36,7 @@ ENV CMAKE_GENERATOR="Ninja"
 ENV CMAKE_EXPORT_COMPILE_COMMANDS="On"
 
 # Install clang toolchain
-COPY .devcontainer/apt-requirements-clang.json /tmp/apt-requirements-clang.json
+COPY .devcontainer/cpp/apt-requirements-clang.json /tmp/apt-requirements-clang.json
 # hadolint ignore=SC1091
 RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot-keyring.gpg \
  && UBUNTU_CODENAME=$(. /etc/os-release; echo "${UBUNTU_CODENAME/*, /}") \

diff --git a/.devcontainer/apt-requirements-base.json → .devcontainer/cpp/apt-requirements-base.json b/.devcontainer/apt-requirements-base.json → .devcontainer/cpp/apt-requirements-base.json
diff --git a/.devcontainer/apt-requirements-clang.json → ...container/cpp/apt-requirements-clang.json b/.devcontainer/apt-requirements-clang.json → ...container/cpp/apt-requirements-clang.json
diff --git a/...ntainer/devcontainer-metadata-vscode.json → ...ner/cpp/devcontainer-metadata-vscode.json b/...ntainer/devcontainer-metadata-vscode.json → ...ner/cpp/devcontainer-metadata-vscode.json
diff --git a/.devcontainer/devcontainer.json → .devcontainer/cpp/devcontainer.json b/.devcontainer/devcontainer.json → .devcontainer/cpp/devcontainer.json
@@ -1,7 +1,10 @@
 {
   "build": {
     "dockerfile": "Dockerfile",
-    "context": ".."
+    "context": "../.."
+  },
+  "remoteEnv": {
+    "CONTAINER_FLAVOR": "cpp"
   },
   "mounts": [
     "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
@@ -14,9 +17,11 @@
         "[email protected]",
         "[email protected]",
         "[email protected]",
+        "[email protected]",
         "[email protected]",
         "[email protected]",
-        "[email protected]"
+        "[email protected]",
+        "[email protected]"
       ]
     }
   }

diff --git a/.devcontainer/requirements.in → .devcontainer/cpp/requirements.in b/.devcontainer/requirements.in → .devcontainer/cpp/requirements.in
diff --git a/.devcontainer/requirements.txt → .devcontainer/cpp/requirements.txt b/.devcontainer/requirements.txt → .devcontainer/cpp/requirements.txt
diff --git a/test/CMakeLists.txt → .devcontainer/cpp/test/CMakeLists.txt b/test/CMakeLists.txt → .devcontainer/cpp/test/CMakeLists.txt
@@ -6,9 +6,8 @@ include(GoogleTest)
 
 add_subdirectory(clang-cl)
 add_subdirectory(clang-tools)
-add_subdirectory(coverage)
 add_subdirectory(fuzzing)
 add_subdirectory(gcc)
 add_subdirectory(gcc-arm-none-eabi)
-add_subdirectory(mutation)
 add_subdirectory(sanitizers)
+add_subdirectory(test)
diff --git a/test/CMakePresets.json → .devcontainer/cpp/test/CMakePresets.json b/test/CMakePresets.json → .devcontainer/cpp/test/CMakePresets.json
diff --git a/test/clang-cl/CMakeLists.txt → ...ontainer/cpp/test/clang-cl/CMakeLists.txt b/test/clang-cl/CMakeLists.txt → ...ontainer/cpp/test/clang-cl/CMakeLists.txt
diff --git a/test/clang-cl/main.cpp → .devcontainer/cpp/test/clang-cl/main.cpp b/test/clang-cl/main.cpp → .devcontainer/cpp/test/clang-cl/main.cpp
diff --git a/test/clang-cl/toolchain.cmake → ...ntainer/cpp/test/clang-cl/toolchain.cmake b/test/clang-cl/toolchain.cmake → ...ntainer/cpp/test/clang-cl/toolchain.cmake
diff --git a/test/clang-tools/CMakeLists.txt → ...ainer/cpp/test/clang-tools/CMakeLists.txt b/test/clang-tools/CMakeLists.txt → ...ainer/cpp/test/clang-tools/CMakeLists.txt
diff --git a/test/clang-tools/main.cpp → .devcontainer/cpp/test/clang-tools/main.cpp b/test/clang-tools/main.cpp → .devcontainer/cpp/test/clang-tools/main.cpp
diff --git a/test/clang-tools/unformatted.cpp → ...iner/cpp/test/clang-tools/unformatted.cpp b/test/clang-tools/unformatted.cpp → ...iner/cpp/test/clang-tools/unformatted.cpp
diff --git a/test/coverage/CMakeLists.txt → ...ontainer/cpp/test/coverage/CMakeLists.txt b/test/coverage/CMakeLists.txt → ...ontainer/cpp/test/coverage/CMakeLists.txt
diff --git a/test/coverage/test.cpp → .devcontainer/cpp/test/coverage/test.cpp b/test/coverage/test.cpp → .devcontainer/cpp/test/coverage/test.cpp
diff --git a/test/fuzzing/CMakeLists.txt → ...container/cpp/test/fuzzing/CMakeLists.txt b/test/fuzzing/CMakeLists.txt → ...container/cpp/test/fuzzing/CMakeLists.txt
diff --git a/test/fuzzing/main.cpp → .devcontainer/cpp/test/fuzzing/main.cpp b/test/fuzzing/main.cpp → .devcontainer/cpp/test/fuzzing/main.cpp
diff --git a/test/gcc-arm-none-eabi/CMakeLists.txt → ...cpp/test/gcc-arm-none-eabi/CMakeLists.txt b/test/gcc-arm-none-eabi/CMakeLists.txt → ...cpp/test/gcc-arm-none-eabi/CMakeLists.txt
diff --git a/test/gcc-arm-none-eabi/main.cpp → ...ainer/cpp/test/gcc-arm-none-eabi/main.cpp b/test/gcc-arm-none-eabi/main.cpp → ...ainer/cpp/test/gcc-arm-none-eabi/main.cpp
diff --git a/test/gcc-arm-none-eabi/toolchain.cmake → ...pp/test/gcc-arm-none-eabi/toolchain.cmake b/test/gcc-arm-none-eabi/toolchain.cmake → ...pp/test/gcc-arm-none-eabi/toolchain.cmake
diff --git a/test/gcc/CMakeLists.txt → .devcontainer/cpp/test/gcc/CMakeLists.txt b/test/gcc/CMakeLists.txt → .devcontainer/cpp/test/gcc/CMakeLists.txt
diff --git a/test/gcc/main-fail.cpp → .devcontainer/cpp/test/gcc/main-fail.cpp b/test/gcc/main-fail.cpp → .devcontainer/cpp/test/gcc/main-fail.cpp
diff --git a/test/gcc/main.cpp → .devcontainer/cpp/test/gcc/main.cpp b/test/gcc/main.cpp → .devcontainer/cpp/test/gcc/main.cpp
diff --git a/test/sanitizers/CMakeLists.txt → ...tainer/cpp/test/sanitizers/CMakeLists.txt b/test/sanitizers/CMakeLists.txt → ...tainer/cpp/test/sanitizers/CMakeLists.txt
diff --git a/test/sanitizers/asan.cpp → .devcontainer/cpp/test/sanitizers/asan.cpp b/test/sanitizers/asan.cpp → .devcontainer/cpp/test/sanitizers/asan.cpp
diff --git a/test/sanitizers/memsan.cpp → .devcontainer/cpp/test/sanitizers/memsan.cpp b/test/sanitizers/memsan.cpp → .devcontainer/cpp/test/sanitizers/memsan.cpp
diff --git a/test/sanitizers/threadsan.cpp → ...ntainer/cpp/test/sanitizers/threadsan.cpp b/test/sanitizers/threadsan.cpp → ...ntainer/cpp/test/sanitizers/threadsan.cpp
diff --git a/test/sanitizers/ubsan.cpp → .devcontainer/cpp/test/sanitizers/ubsan.cpp b/test/sanitizers/ubsan.cpp → .devcontainer/cpp/test/sanitizers/ubsan.cpp
diff --git a/.devcontainer/cpp/test/test/CMakeLists.txt b/.devcontainer/cpp/test/test/CMakeLists.txt
@@ -0,0 +1,31 @@
+option(ENABLE_COVERAGE_TEST Off)
+option(ENABLE_MUTATION_TESTING_TEST Off)
+
+include(FetchContent)
+
+FetchContent_Declare(googletest
+    GIT_REPOSITORY https://github.com/google/googletest
+    GIT_TAG v1.14.0
+)
+
+if (ENABLE_COVERAGE_TEST)
+    FetchContent_MakeAvailable(googletest)
+
+    add_executable(test-coverage EXCLUDE_FROM_ALL test.cpp)
+    target_compile_options(test-coverage PRIVATE
+        -g -O0 --coverage -fprofile-arcs -ftest-coverage -fno-inline
+    )
+    target_link_libraries(test-coverage PRIVATE gmock_main gcov)
+    add_test(NAME test-coverage COMMAND test-coverage)
+endif()
+
+if (ENABLE_MUTATION_TESTING_TEST)
+    FetchContent_MakeAvailable(googletest)
+
+    add_executable(test-mutation EXCLUDE_FROM_ALL test.cpp)
+    target_compile_options(test-mutation PRIVATE
+        -g -O0 -grecord-command-line -fprofile-instr-generate -fcoverage-mapping -fpass-plugin=/usr/lib/mull-ir-frontend
+    )
+    target_link_libraries(test-mutation PRIVATE gmock_main)
+    add_test(NAME test-mutation COMMAND mull-runner $<TARGET_FILE:test-mutation>)
+endif()
diff --git a/test/mutation/test.cpp → .devcontainer/cpp/test/test/test.cpp b/test/mutation/test.cpp → .devcontainer/cpp/test/test/test.cpp
diff --git a/test/testsuite.bats → .devcontainer/cpp/test/testsuite.bats b/test/testsuite.bats → .devcontainer/cpp/test/testsuite.bats
@@ -32,149 +32,105 @@ teardown() {
   rm -rf build crash-*
 }
 
-# bats test_tags=tc:1
 @test "valid code input should result in working executable using host compiler" {
-  run cmake --preset gcc
-  assert_success
-
-  run cmake --build --preset gcc
-  assert_success
+  cmake --preset gcc
+  cmake --build --preset gcc
 
   run build/gcc/gcc/test-gcc
   assert_success
   assert_output "Hello World!"
 }
 
-# bats test_tags=tc:2
 @test "valid code input should result in elf executable using arm-none-eabi compiler" {
-  run cmake --preset gcc-arm-none-eabi
-  assert_success
-
-  run cmake --build --preset gcc-arm-none-eabi
-  assert_success
+  cmake --preset gcc-arm-none-eabi
+  cmake --build --preset gcc-arm-none-eabi
 
   run readelf -h build/gcc-arm-none-eabi/gcc-arm-none-eabi/test-gcc-arm-none-eabi
   assert_output --partial "Type:                              EXEC"
   assert_output --partial "Machine:                           ARM"
 }
 
-# bats test_tags=tc:3
-@test "valid code input should result in working Windows executable using clang-cl compiler" {
-  run cmake --preset clang-cl
-  assert_success
-
-  run cmake --build --preset clang-cl
-  assert_success
+@test "valid code input should result in Windows executable using clang-cl compiler" {
+  cmake --preset clang-cl
+  cmake --build --preset clang-cl
 }
 
-# bats test_tags=tc:20
 @test "compilation database should be generated on CMake configure" {
-  run cmake --preset gcc
-  [ -e build/gcc/compile_commands.json ]
+  cmake --preset gcc
+  assert [ -e build/gcc/compile_commands.json ]
 
-  run cmake --preset gcc-arm-none-eabi
-  [ -e build/gcc-arm-none-eabi/compile_commands.json ]
+  cmake --preset gcc-arm-none-eabi
+  assert [ -e build/gcc-arm-none-eabi/compile_commands.json ]
 }
 
-# bats test_tags=tc:4
 @test "invalid code input should result in failing build" {
-  run cmake --preset gcc
-  assert_success
-
-  run cmake --build --preset gcc-fail
-  assert_failure
+  cmake --preset gcc
+  run ! cmake --build --preset gcc-fail
 }
 
-# bats test_tags=tc:5
 @test "using ccache as a compiler launcher should result in cached build using gcc compiler" {
-  run ccache --clear --zero-stats
-
-  run cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-  assert_success
-
-  run cmake --build --preset gcc
-  assert_success
+  ccache --clear --zero-stats
+  cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  cmake --build --preset gcc
 
   run ccache -s
   assert_output --partial "Hits:               0"
   assert_output --partial "Misses:             1"
 
-  run rm -rf build
-  run ccache --zero-stats
-
-  run cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-  assert_success
-
-  run cmake --build --preset gcc
-  assert_success
+  rm -rf build
+  ccache --zero-stats
+  cmake --preset gcc -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  cmake --build --preset gcc
 
   run ccache -s
   assert_output --partial "Hits:               1"
   assert_output --partial "Misses:             0"
 }
 
-# bats test_tags=tc:17
 @test "using ccache as a compiler launcher should result in cached build using clang-cl compiler" {
-  run ccache --clear --zero-stats
-
-  run cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-  assert_success
-
-  run cmake --build --preset clang-cl
-  assert_success
+  ccache --clear --zero-stats
+  cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  cmake --build --preset clang-cl
 
   run ccache -s
   assert_output --partial "Hits:               0"
   assert_output --partial "Misses:             1"
 
-  run rm -rf build
-  run ccache --zero-stats
-
-  run cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
-  assert_success
-
-  run cmake --build --preset clang-cl
-  assert_success
+  rm -rf build
+  ccache --zero-stats
+  cmake --preset clang-cl -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+  cmake --build --preset clang-cl
 
   run ccache -s
   assert_output --partial "Hits:               1"
   assert_output --partial "Misses:             0"
 }
 
-# bats test_tags=tc:6
 @test "running clang-tidy as part of the build should result in warning diagnostics" {
-  run cmake --preset clang
-  assert_success
+  cmake --preset clang
 
   run cmake --build --preset clang-tidy
   assert_success
   assert_output --partial "warning: use a trailing return type for this function"
 }
 
-# bats test_tags=tc:7
 @test "running include-what-you-use as part of the build should result in warning diagnostics" {
-  run cmake --preset clang
-  assert_success
+  cmake --preset clang
 
   run cmake --build --preset clang-iwyu
   assert_success
   assert_output --partial "Warning: include-what-you-use reported diagnostics:"
 }
 
-# bats test_tags=tc:8
 @test "running clang-format should result in re-formatted code" {
   run clang-format clang-tools/unformatted.cpp
   assert_success
   assert_output "int main() {}"
 }
 
-# bats test_tags=tc:9
 @test "coverage information should be generated when running a testsuite" {
-  run cmake --preset coverage
-  assert_success
-
-  run cmake --build --preset coverage
-  assert_success
+  cmake --preset coverage
+  cmake --build --preset coverage
 
   run ctest --preset coverage
   assert_success
@@ -185,57 +141,40 @@ teardown() {
   assert_output --partial "GCC Code Coverage Report"
 }
 
-# bats test_tags=tc:10
 @test "crashes should be detected when fuzzing an executable" {
-  run cmake --preset clang
-  assert_success
-
-  run cmake --build --preset fuzzing
-  assert_success
+  cmake --preset clang
+  cmake --build --preset fuzzing
 
   run build/clang/fuzzing/test-fuzzing
   assert_failure
   assert_output --partial "SUMMARY: libFuzzer: deadly signal"
 }
 
-# bats test_tags=tc:11
 @test "a mutation score should be calculated when mutation testing a testsuite" {
-  run cmake --preset mutation
-  assert_success
-
-  run cmake --build --preset mutation
-  assert_success
+  cmake --preset mutation
+  cmake --build --preset mutation
 
   run ctest --preset mutation
   assert_output --partial "[info] Mutation score:"
 }
 
-# bats test_tags=tc:12
 @test "host gdb should be able to start" {
-  run gdb --version
-  assert_success
+  gdb --version
 }
 
-# bats test_tags=tc:13
 @test "gdb-multiarch should be able to start" {
-  run gdb-multiarch --version
-  assert_success
+  gdb-multiarch --version
 }
 
-# bats test_tags=tc:14
 @test "clangd should be able to analyze source files" {
   run clangd --check=gcc/main.cpp
   assert_success
   assert_output --partial "All checks completed, 0 errors"
 }
 
-# bats test_tags=tc:15
 @test "using lld as an alternative linker should result in working host executable" {
-  run cmake --preset gcc
-  assert_success
-
-  run cmake --build --preset gcc-lld
-  assert_success
+  cmake --preset gcc
+  cmake --build --preset gcc-lld
 
   run readelf --string-dump .comment build/gcc/gcc/test-gcc-lld
   assert_output --partial "Linker: Ubuntu LLD"
@@ -245,18 +184,15 @@ teardown() {
   assert_output "Hello World!"
 }
 
-# bats test_tags=tc:18
 @test "when the docker socket is mounted, using the docker cli should give access to the host docker daemon" {
   run docker info
   assert_success
   assert_output --partial "Server Version:"
 }
 
-# bats test_tags=tc:21
 @test "sanitizers should detect undefined or suspicious behavior in code compiled with gcc" {
-  run cmake --preset gcc
-  run cmake --build --preset gcc-sanitizers
-  assert_success
+  cmake --preset gcc
+  cmake --build --preset gcc-sanitizers
 
   run build/gcc/sanitizers/test-asan
   assert_failure
@@ -267,11 +203,9 @@ teardown() {
   assert_output --partial "runtime error: load of null pointer"
 }
 
-# bats test_tags=tc:22
 @test "sanitizers should detect undefined or suspicious behavior in code compiled with clang" {
-  run cmake --preset clang
-  run cmake --build --preset clang-sanitizers
-  assert_success
+  cmake --preset clang
+  cmake --build --preset clang-sanitizers
 
   run build/clang/sanitizers/test-asan
   assert_failure