From 0b27564a54e85f3b5a2c7243bfd7d2e386569682 Mon Sep 17 00:00:00 2001 From: Nardi Ivan Date: Mon, 17 Jun 2024 11:09:13 +0200 Subject: [PATCH] Make the CI faster Without the `-fsanitize-memory-track-origins` flag, MSAN job is ~30% faster. Since this flag is useful only while debugging (and not to simply discover memory issues), avoid it on the CI. Note that, by default it is still enabled by default. Right now, MingW runs on *every* ubuntu builds: limit it only to the standard matrix (i.e. ubuntu 20.04, 22.04, 24.04 with default configuration), without any sanitizers (note that MingW doesn't support *san anyway). armhf job is by far the longest job in the CI: remove asan configuration to make it faster. Note that we already have a lot of different jobs (on x86_64) with some sanitizers, and that the other 2 jobs on arm/s390x don't have asan support anyway. If we really, really want a job with arm + asan we can add it as a async/scheduled job. Remove an old workaround for ubuntu jobs Avoid installing packages needed only for the documentation About `check_symbols.sh` script: even if uses the compiled library/objects, it basicaly only checks if we are using, in the source code, same functions that we shoudn't. We don't need to perform the same kind of check so many times.. --- .github/workflows/build.yml | 20 ++++++++------------ .github/workflows/cifuzz.yml | 2 +- configure.ac | 6 +++++- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e35e9841e52..8c57a2727fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -172,7 +172,7 @@ jobs: gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-memory-sanitizer" + msan: "--with-memory-sanitizer --disable-memory-track-origins" nBPF: "" - compiler: "cc" os: macOS-13 @@ -204,7 +204,7 @@ jobs: gcrypt: "" pcre: "--with-pcre2" maxminddb: "--with-maxminddb" - msan: "--with-sanitizer" + msan: "" nBPF: "" - compiler: "cc" os: ubuntu-latest @@ -222,20 +222,16 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis - - name: Fix kernel mmap rnd bits on Ubuntu - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') - run: | - # Workaround for compatinility between latest kernel and sanitizer - # See https://github.com/actions/runner-images/issues/9491 - sudo sysctl vm.mmap_rnd_bits=28 - name: Install Ubuntu Prerequisites if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') run: | sudo apt-get update sudo apt-get install autoconf automake libtool pkg-config gettext libjson-c-dev flex bison libpcap-dev - sudo apt-get install gcc-mingw-w64 libc6-dev - sudo apt-get install doxygen python3-sphinx python3-sphinx-rtd-theme python3-breathe python3-pip sudo apt-get install rrdtool librrd-dev parallel + - name: Install Ubuntu Prerequisites [Mingw-w64] (runs only on ubuntu jobs) + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers + run: | + sudo apt-get install gcc-mingw-w64 libc6-dev - name: Install Ubuntu Prerequisites (libgcrypt) if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && startsWith(matrix.gcrypt, '--with-local-libgcrypt') run: | @@ -333,7 +329,7 @@ jobs: DESTDIR=/tmp/ndpi make install ls -alhHR /tmp/ndpi - name: Test nDPI [SYMBOLS] - if: (startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'mac')) && startsWith(matrix.arch, 'x86_64') + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers run: | ./utils/check_symbols.sh || { FAILED=$?; echo "::error file=${NDPI_LIB}::Unwanted libc symbols found: ${FAILED}. Please make sure to use only ndpi_malloc/ndpi_calloc/ndpi_realloc/ndpi_free wrapper instead of malloc/calloc/realloc/free."; false; } env: @@ -369,7 +365,7 @@ jobs: make dist ./utils/verify_dist_tarball.sh - name: Build nDPI [Mingw-w64] (runs only on ubuntu jobs) - if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.nBPF, 'nBPF') + if: startsWith(matrix.os, 'ubuntu') && startsWith(matrix.arch, 'x86_64') && !startsWith(matrix.msan, '--with-') #Only on a few "standard" builds, without any sanitizers run: | make distclean ./autogen.sh --enable-option-checking=fatal --enable-debug-messages --enable-tls-sigs --host=x86_64-w64-mingw32 diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 9f4ac23a95d..de65fccb6c0 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -18,7 +18,7 @@ jobs: uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master with: oss-fuzz-project-name: 'ndpi' - fuzz-seconds: 1320 + fuzz-seconds: 1200 dry-run: false sanitizer: ${{ matrix.sanitizer }} - name: Check Crash (fails when a crash is detected) diff --git a/configure.ac b/configure.ac index 7b2f99eba39..f3037d0aa9b 100644 --- a/configure.ac +++ b/configure.ac @@ -30,6 +30,7 @@ AC_ARG_WITH(nbpf-path, AS_HELP_STRING([--with-nbpf-path], [nBPF library custom p AC_ARG_WITH(lto-and-gold-linker, AS_HELP_STRING([--with-lto-and-gold-linker], [Build with LTO and Gold linker])) AC_ARG_ENABLE(debug-build, AS_HELP_STRING([--enable-debug-build], [Enable debug build (`-g` flag)]),[enable_debugbuild=$enableval],[enable_debugbuild=no]) AC_ARG_ENABLE(global-context-support, AS_HELP_STRING([--disable-global-context-support], [Disable support for global context. No external dependency on libpthread])) +AC_ARG_ENABLE(memory-track-origins, AS_HELP_STRING([--disable-memory-track-origins], [Don't add -fsanitize-memory-track-origins flag when compiling with MASAN support. Useful for faster CI])) NDPI_CFLAGS="${NDPI_CFLAGS} -D_DEFAULT_SOURCE=1 -D_GNU_SOURCE=1" @@ -81,7 +82,10 @@ AS_IF([test "${with_thread_sanitizer+set}" = set],[ ]) AS_IF([test "${with_memory_sanitizer+set}" = set],[ - NDPI_CFLAGS="${NDPI_CFLAGS} -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer" + NDPI_CFLAGS="${NDPI_CFLAGS} -fsanitize=memory -fno-omit-frame-pointer" + AS_IF([test "x$enable_memory_track_origins" != "xno"], [ + NDPI_CFLAGS="${NDPI_CFLAGS} -fsanitize-memory-track-origins" + ]) NDPI_LDFLAGS="${NDPI_LDFLAGS} -fsanitize=memory" ])