From d2a3e060926d87759ffb8374da4c74cadbe4f7db Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 10 Nov 2021 12:50:52 +0100 Subject: [PATCH 01/15] update afl++ commit id --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 256e7be56ff2..3fcfa26fbbba 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 4fe572b80f76ff0b0e916b639d1e04d5af48b157 + git checkout 533e979010ca338df6fc415d87668f8187752915 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From c4e7ba2cd05553c5ad7fd56d5bd88a44bd5b155a Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 17 Nov 2021 16:00:02 +0100 Subject: [PATCH 02/15] update afl++ commit id --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 3fcfa26fbbba..1bd569246900 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 533e979010ca338df6fc415d87668f8187752915 + git checkout 132630d48d0f9fe50e9388f941433c85636587da RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From b9230814888dc6e4bb167770a6010ae1007501ac Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 19 Nov 2021 17:01:37 +0100 Subject: [PATCH 03/15] fix for afl++ --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 1bd569246900..d066e2e013a9 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 132630d48d0f9fe50e9388f941433c85636587da + git checkout 7777045c09c404b1274c930788317525fedb43ad RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From 5622a2af6cb898127db03c3ca787481f065e2bea Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 22 Nov 2021 14:43:09 +0100 Subject: [PATCH 04/15] attempt fix for curl --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index d066e2e013a9..34b14708c9e9 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 7777045c09c404b1274c930788317525fedb43ad + git checkout 6f9a98c4a97e8e261fc52891d61f0b0c145b6364 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From 99e70ddeb860b1ea1abb90864e6f65481b73847f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 Nov 2021 17:15:40 +0100 Subject: [PATCH 05/15] allow easy reproducable afl++ builds --- infra/base-images/base-builder/compile_afl | 97 ++++++++++++---------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl index d6509c74c7a6..a9e5ae250b5f 100644 --- a/infra/base-images/base-builder/compile_afl +++ b/infra/base-images/base-builder/compile_afl @@ -15,20 +15,10 @@ # ################################################################################ -# afl++ configuration options. -# The 'env|grep' setup ensures we do not trigger the linter. -# The variables need to be set to "1" here - or before running this script. - -# AFL++ settings. -export AFL_LLVM_MODE_WORKAROUND=0 -export AFL_ENABLE_DICTIONARY=0 -export AFL_ENABLE_CMPLOG=1 -export AFL_LAF_CHANCE=3 +# AFL++ setup +echo "Copying precompiled AFL++" -# Start compiling afl++. -echo "Copying precompiled afl++" - -# Copy afl++ tools necessary for fuzzing. +# Copy AFL++ tools necessary for fuzzing. pushd $SRC/aflplusplus > /dev/null cp -f libAFLDriver.a $LIB_FUZZING_ENGINE @@ -39,42 +29,65 @@ ls afl-* *.txt *.a *.o *.so | sort -u | xargs cp -t $OUT export CC="$SRC/aflplusplus/afl-clang-fast" export CXX="$SRC/aflplusplus/afl-clang-fast++" -# Set sane afl++ environment defaults: +# Set sane AFL++ environment defaults: # Be quiet, otherwise this can break some builds. export AFL_QUIET=1 # No leak errors during builds. export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1" +# No complain on unknown AFL environment variables +export AFL_IGNORE_UNKNOWN_ENVS=1 -# AFL compile option roulette. It is OK if they all happen together. +# To analyze build failures and set specific AFL++ settings, set +# `export AFL_SKIP_OSSFUZZ=1` +# The 'env|grep' setup ensures we do not trigger the linter. +env | egrep -q '^AFL_SKIP_OSSFUZZ=' || { + + # The variables need to be set to "1" here - or before running this script. + # AFL++ configuration options. + export AFL_LLVM_MODE_WORKAROUND=0 + export AFL_ENABLE_DICTIONARY=0 + export AFL_ENABLE_CMPLOG=1 + export AFL_LAF_CHANCE=5 + + # + # AFL++ compile option roulette. It is OK if they all happen together. + # + + # 20% chance for CTX-2 coverage instrumentation (Caller conTeXt sensitive + # edge coverage). + test $(($RANDOM % 100)) -lt 20 && { + export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2 + export AFL_ENABLE_CMPLOG=0 + # we increase the chance for LAF because we do not do CMPLOG with CTX + export AFL_LAF_CHANCE=30 + } -# 20% chance for CTX-2 coverage instrumentation (Caller conTeXt sensitive -# edge coverage). -test $(($RANDOM % 100)) -lt 20 && { - export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2 - export AFL_ENABLE_CMPLOG=0 - export AFL_LAF_CHANCE=30 -} + # 40% chance to create a dictionary. + test $(($RANDOM % 100)) -lt 40 && { + export AFL_ENABLE_DICTIONARY=1 + } -# 40% chance to create a dictionary. -test $(($RANDOM % 100)) -lt 40 && { - export AFL_ENABLE_DICTIONARY=1 -} + # 60% chance to perform CMPLOG/REDQUEEN. + rm -f "$OUT/afl_cmplog.txt" + test "$AFL_ENABLE_CMPLOG" = "1" -a $(($RANDOM % 100)) -lt 60 && { + export AFL_LLVM_CMPLOG=1 + touch "$OUT/afl_cmplog.txt" + } -# 60% chance to perform CMPLOG/REDQUEEN. -rm -f "$OUT/afl_cmplog.txt" -test "$AFL_ENABLE_CMPLOG" = "1" -a $(($RANDOM % 100)) -lt 60 && { - export AFL_LLVM_CMPLOG=1 - touch "$OUT/afl_cmplog.txt" -} + # chance to perform COMPCOV/LAF_INTEL - if CMPLOG is not enabled. + test $(($RANDOM % 100)) -lt $AFL_LAF_CHANCE -a "$AFL_ENABLE_CMPLOG" = "0" && { + export AFL_LLVM_LAF_ALL=1 + } -# 3% chance to perform COMPCOV/LAF_INTEL. -test $(($RANDOM % 100)) -lt $AFL_LAF_CHANCE && { - export AFL_LLVM_LAF_ALL=1 -} + # + # End of AFL++ compile option roulette + # + + # Create a dictionary if one is wanted. + test "$AFL_ENABLE_DICTIONARY" = "1" && { + export AFL_LLVM_DICT2FILE="$OUT/afl++.dict" + } -# Create a dictionary if one is wanted. -test "$AFL_ENABLE_DICTIONARY" = "1" && { - export AFL_LLVM_DICT2FILE="$OUT/afl++.dict" } # In case afl-clang-fast ever breaks, this is a workaround: @@ -96,10 +109,10 @@ test "$AFL_LLVM_MODE_WORKAROUND" = "1" && { cp -f libAFLDrivernew.a $LIB_FUZZING_ENGINE } -# Provide a way to document the afl++ options used in this build: +# Provide a way to document the AFL++ options used in this build: echo -echo afl++ target compilation setup: -env | grep AFL_ | tee "$OUT/afl_options.txt" +echo AFL++ target compilation setup: +env | egrep '^AFL_' | tee "$OUT/afl_options.txt" echo popd > /dev/null From 591f905456773fdf3d6937797a6c0922c14847a1 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 30 Nov 2021 14:46:07 +0100 Subject: [PATCH 06/15] new commit id --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 34b14708c9e9..475e5a7d1134 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 6f9a98c4a97e8e261fc52891d61f0b0c145b6364 + git checkout 5525f8c9ef8bb879dadd0eb942d524827d1b0362 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From 621a60a685f14d41f71407844b5e450b59c0634b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 11 Dec 2021 14:22:50 +0100 Subject: [PATCH 07/15] fixes --- infra/base-images/base-builder/Dockerfile | 2 +- infra/base-images/base-builder/compile_afl | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 475e5a7d1134..cb3de3185c33 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 5525f8c9ef8bb879dadd0eb942d524827d1b0362 + git checkout 5ec91ad5291228f6ef6d9b69605a9b752ef28fa0 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl index a9e5ae250b5f..6a8341431a34 100644 --- a/infra/base-images/base-builder/compile_afl +++ b/infra/base-images/base-builder/compile_afl @@ -37,6 +37,9 @@ export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_ # No complain on unknown AFL environment variables export AFL_IGNORE_UNKNOWN_ENVS=1 +# To not use AFL++'s afl-cc compiler but clang sancov, set this to 1: +export AFL_LLVM_MODE_WORKAROUND=0 + # To analyze build failures and set specific AFL++ settings, set # `export AFL_SKIP_OSSFUZZ=1` # The 'env|grep' setup ensures we do not trigger the linter. @@ -44,7 +47,6 @@ env | egrep -q '^AFL_SKIP_OSSFUZZ=' || { # The variables need to be set to "1" here - or before running this script. # AFL++ configuration options. - export AFL_LLVM_MODE_WORKAROUND=0 export AFL_ENABLE_DICTIONARY=0 export AFL_ENABLE_CMPLOG=1 export AFL_LAF_CHANCE=5 From 69fd0a106f7450873c87ea9e64db6ec765c929e9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 11 Dec 2021 17:45:15 +0100 Subject: [PATCH 08/15] another afl++ commit increase --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index cb3de3185c33..78f2180225f2 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 5ec91ad5291228f6ef6d9b69605a9b752ef28fa0 + git checkout 34c9654690907fa4c0de4073acdc198a4135c4f9 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From 76560c745871cd10736b5f5bc78e1c6b047b42ff Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 13 Dec 2021 18:31:50 +0100 Subject: [PATCH 09/15] not a relevant update, but why not --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 78f2180225f2..52e2b4268bac 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 34c9654690907fa4c0de4073acdc198a4135c4f9 + git checkout 52dd5d479d411460c8cf2010a89f72e269d799ea RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From 0770fcf0c6968c737be0dc6f66d4f9daa2f1407d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 16 Dec 2021 10:15:05 +0100 Subject: [PATCH 10/15] two afl++ fixes --- infra/base-images/base-builder/Dockerfile | 2 +- infra/base-images/base-builder/compile_afl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 475e5a7d1134..41889e7ae36a 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 5525f8c9ef8bb879dadd0eb942d524827d1b0362 + git checkout 5f70bc54043a47c232be83ca77f53ddb6bb81908 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl index a9e5ae250b5f..9275fd114a27 100644 --- a/infra/base-images/base-builder/compile_afl +++ b/infra/base-images/base-builder/compile_afl @@ -59,7 +59,7 @@ env | egrep -q '^AFL_SKIP_OSSFUZZ=' || { export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2 export AFL_ENABLE_CMPLOG=0 # we increase the chance for LAF because we do not do CMPLOG with CTX - export AFL_LAF_CHANCE=30 + export AFL_LAF_CHANCE=20 } # 40% chance to create a dictionary. From 5e0a959743e85ec0844fc88e56a0d6e748175f82 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 16 Dec 2021 10:39:51 +0100 Subject: [PATCH 11/15] add debug script --- infra/base-images/base-builder/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 41889e7ae36a..5f07761c2e3c 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -122,10 +122,11 @@ ENV FUZZER_LDFLAGS "" WORKDIR $SRC -# TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 5f70bc54043a47c232be83ca77f53ddb6bb81908 + git checkout 5f70bc54043a47c232be83ca77f53ddb6bb81908 && \ + wget --no-check-certificate -O oss.sh https://raw.githubusercontent.com/vanhauser-thc/binary_blobs/master/oss.sh && \ + chmod 755 oss.sh RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ From 38463dc2784e488583327005c19576af75dec3c1 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 16 Dec 2021 10:53:27 +0100 Subject: [PATCH 12/15] update commit id (not relevant though) --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 5f07761c2e3c..e4237e4619f8 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -124,7 +124,7 @@ WORKDIR $SRC RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 5f70bc54043a47c232be83ca77f53ddb6bb81908 && \ + git checkout 3cb7319ccdb98dcc6b023dbead603a4450ac4541 && \ wget --no-check-certificate -O oss.sh https://raw.githubusercontent.com/vanhauser-thc/binary_blobs/master/oss.sh && \ chmod 755 oss.sh From 618e305851391f04dc57fe93f6a6d70b28e42d71 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 16 Dec 2021 11:03:15 +0100 Subject: [PATCH 13/15] add wget --- infra/base-images/base-builder/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index e4237e4619f8..b4cf9a84b665 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -25,6 +25,7 @@ RUN dpkg --add-architecture i386 && \ binutils-dev \ build-essential \ curl \ + wget \ git \ jq \ libc6-dev-i386 \ From a959484784f7be6087dc81845bfcf741c46816e4 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 4 Jan 2022 17:29:13 +0100 Subject: [PATCH 14/15] update afl++ commit id --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index b4cf9a84b665..6bcb5f95dc12 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 3cb7319ccdb98dcc6b023dbead603a4450ac4541 && \ + git checkout ee295801a69d10121dd94ebfb4db9308085de6e4 && \ wget --no-check-certificate -O oss.sh https://raw.githubusercontent.com/vanhauser-thc/binary_blobs/master/oss.sh && \ chmod 755 oss.sh From 40ab765384217ac4b3f4c60655dc4b2d839e376d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 11 Jan 2022 10:24:47 +0100 Subject: [PATCH 15/15] just afl++ doc updates, but maybe this results in some attention --- infra/base-images/base-builder/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 6bcb5f95dc12..1f6018e06238 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -125,7 +125,7 @@ WORKDIR $SRC RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout ee295801a69d10121dd94ebfb4db9308085de6e4 && \ + git checkout ef77d552e9b89852b293c300a5a9a9c617945f43 && \ wget --no-check-certificate -O oss.sh https://raw.githubusercontent.com/vanhauser-thc/binary_blobs/master/oss.sh && \ chmod 755 oss.sh