Merge pull request AFLplusplus#1592 from AFLplusplus/dev

Dev

GNUmakefile

@@ -91,9 +91,8 @@ ifneq "$(SYS)" "Darwin"
   #ifeq "$(HAVE_MARCHNATIVE)" "1"
   #  SPECIAL_PERFORMANCE += -march=native
   #endif
- # OS X does not like _FORTIFY_SOURCE=2
  ifndef DEBUG
-   CFLAGS_OPT += -D_FORTIFY_SOURCE=2
+   CFLAGS_OPT += -D_FORTIFY_SOURCE=1
  endif
 else
   # On some odd MacOS system configurations, the Xcode sdk path is not set correctly
@@ -103,7 +102,7 @@ endif
 
 ifeq "$(SYS)" "SunOS"
   CFLAGS_OPT += -Wno-format-truncation
-  LDFLAGS = -lkstat -lrt
+  LDFLAGS = -lkstat -lrt -lsocket -lnsl
 endif
 
 ifdef STATIC

GNUmakefile.gcc_plugin

@@ -28,14 +28,14 @@ MAN_PATH    ?= $(PREFIX)/share/man/man8
 
 VERSION     = $(shell grep '^$(HASH)define VERSION ' ./config.h | cut -d '"' -f2)
 
-CFLAGS          ?= -O3 -g -funroll-loops -D_FORTIFY_SOURCE=2
+CFLAGS          ?= -O3 -g -funroll-loops -D_FORTIFY_SOURCE=1
 CFLAGS_SAFE     := -Wall -Iinclude -Wno-pointer-sign \
                    -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \
                    -DGCC_VERSION=\"$(GCCVER)\" -DGCC_BINDIR=\"$(GCCBINDIR)\" \
                    -Wno-unused-function
 override CFLAGS += $(CFLAGS_SAFE)
 
-CXXFLAGS    ?= -O3 -g -funroll-loops -D_FORTIFY_SOURCE=2
+CXXFLAGS    ?= -O3 -g -funroll-loops -D_FORTIFY_SOURCE=1
 CXXEFLAGS   := $(CXXFLAGS) -Wall -std=c++11
 
 CC          ?= gcc

GNUmakefile.llvm

@@ -254,7 +254,7 @@ else
         AFL_CLANG_DEBUG_PREFIX =
 endif
 
-CFLAGS          ?= -O3 -funroll-loops -fPIC -D_FORTIFY_SOURCE=2
+CFLAGS          ?= -O3 -funroll-loops -fPIC -D_FORTIFY_SOURCE=1
 CFLAGS_SAFE     := -Wall -g -Wno-cast-qual -Wno-variadic-macros -Wno-pointer-sign \
                    -I ./include/ -I ./instrumentation/ \
                    -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \
@@ -274,9 +274,9 @@ ifdef AFL_TRACE_PC
   $(info Compile option AFL_TRACE_PC is deprecated, just set AFL_LLVM_INSTRUMENT=PCGUARD to activate when compiling targets )
 endif
 
-CXXFLAGS          ?= -O3 -funroll-loops -fPIC -D_FORTIFY_SOURCE=2
+CXXFLAGS          ?= -O3 -funroll-loops -fPIC -D_FORTIFY_SOURCE=1
 override CXXFLAGS += -Wall -g -I ./include/ \
-                     -DVERSION=\"$(VERSION)\" -Wno-variadic-macros \
+                     -DVERSION=\"$(VERSION)\" -Wno-variadic-macros -Wno-deprecated-copy-with-dtor \
                      -DLLVM_MINOR=$(LLVM_MINOR) -DLLVM_MAJOR=$(LLVM_MAJOR)
 
 ifneq "$(shell $(LLVM_CONFIG) --includedir) 2> /dev/null" ""

TODO.md

@@ -2,12 +2,14 @@
 
 ## Should
 
- - better documentation for custom mutators
+ - support afl_custom_{send,post_process}, persistent and deferred fork
+   server in afl-showmap
  - better autodetection of shifting runtime timeout values
  - Update afl->pending_not_fuzzed for MOpt
  - afl-plot to support multiple plot_data
  - parallel builds for source-only targets
  - get rid of check_binary, replace with more forkserver communication
+ - first fuzzer should be a main automatically
 
 ## Maybe
 

docs/Changelog.md

@@ -9,6 +9,9 @@
       send fuzz data to the target as you need, e.g. via IPC.
     - cmplog mode now has -l R option for random colorization, thanks
       to guyf2010 for the PR!
+    - queue statistics are written every 30 minutes to
+      out/NAME/queue_data - likely this will be moved to a debug flag
+      in the future.
   - afl-showmap/afl-cmin
     - -t none now translates to -t 120000 (120 seconds)
   - unicorn_mode updated

docs/env_variables.md

@@ -378,10 +378,10 @@ checks or alter some of the more exotic semantics of the tool:
     valid terminal was detected (for virtual consoles).
 
   - Setting `AFL_FORKSRV_INIT_TMOUT` allows you to specify a different timeout
-    to wait for the forkserver to spin up. The default is the `-t` value times
-    `FORK_WAIT_MULT` from `config.h` (usually 10), so for a `-t 100`, the
-    default would wait for `1000` milliseconds. Setting a different time here is
-    useful if the target has a very slow startup time, for example, when doing
+    to wait for the forkserver to spin up. The specified value is the new timeout, in milliseconds.
+    The default is the `-t` value times `FORK_WAIT_MULT` from `config.h` (usually 10), so for a `-t 100`, the default would wait for `1000` milliseconds.
+    The `AFL_FORKSRV_INIT_TMOUT` value does not get multiplied. It overwrites the initial timeout afl-fuzz waits for the target to come up with a constant time.
+    Setting a different time here is useful if the target has a very slow startup time, for example, when doing
     full-system fuzzing or emulation, but you don't want the actual runs to wait
     too long for timeouts.
 

docs/fuzzing_in_depth.md

@@ -900,6 +900,13 @@ then color-codes the input based on which sections appear to be critical and
 which are not; while not bulletproof, it can often offer quick insights into
 complex file formats.
 
+`casr-afl` from [CASR](https://github.com/ispras/casr) tools provides
+comfortable triaging for crashes found by AFL++. Reports are clustered and
+contain severity and other information.
+```shell
+casr-afl -i /path/to/afl/out/dir -o /path/to/casr/out/dir
+```
+
 ## 5. CI fuzzing
 
 Some notes on continuous integration (CI) fuzzing - this fuzzing is different to

docs/third_party_tools.md

@@ -62,3 +62,5 @@
   generates builds of debian packages suitable for AFL.
 * [afl-fid](https://github.com/FoRTE-Research/afl-fid) - a set of tools for
   working with input data.
+* [CASR](https://github.com/ispras/casr) - a set of tools for crash triage and
+  analysis.

frida_mode/GNUmakefile

@@ -145,7 +145,7 @@ ifndef OS
  $(error "Operating system unsupported")
 endif
 
-GUM_DEVKIT_VERSION=16.0.1
+GUM_DEVKIT_VERSION=16.0.6
 GUM_DEVKIT_FILENAME=frida-gumjs-devkit-$(GUM_DEVKIT_VERSION)-$(OS)-$(ARCH).tar.xz
 GUM_DEVKIT_URL="https://github.com/frida/frida/releases/download/$(GUM_DEVKIT_VERSION)/$(GUM_DEVKIT_FILENAME)"
 
@@ -191,6 +191,9 @@ all: $(FRIDA_TRACE) $(FRIDA_TRACE_LIB) $(AFLPP_FRIDA_DRIVER_HOOK_OBJ) $(AFLPP_QE
 32:
 	CFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all
 
+arm:
+	CFLAGS="-marm" LDFLAGS="-marm" ARCH="armhf" TARGET_CC=arm-linux-gnueabihf-gcc TARGET_CXX=arm-linux-gnueabihf-g++ make all
+
 $(BUILD_DIR):
 	mkdir -p $(BUILD_DIR)
 

frida_mode/src/cmplog/cmplog_arm64.c

@@ -204,10 +204,7 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
 
   gsize address = context->pc;
 
-  register uintptr_t k = (uintptr_t)address;
-
-  k = (k >> 4) ^ (k << 8);
-  k &= CMP_MAP_W - 1;
+  register uintptr_t k = instrument_get_offset_hash(GUM_ADDRESS(address));
 
   if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS)
     __afl_cmp_map->headers[k].hits = 0;

frida_mode/src/cmplog/cmplog_x64.c

@@ -188,10 +188,7 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
 
   gsize address = ctx_read_reg(context, X86_REG_RIP);
 
-  register uintptr_t k = (uintptr_t)address;
-
-  k = (k >> 4) ^ (k << 8);
-  k &= CMP_MAP_W - 7;
+  register uintptr_t k = instrument_get_offset_hash(GUM_ADDRESS(address));
 
   if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS)
     __afl_cmp_map->headers[k].hits = 0;

frida_mode/src/cmplog/cmplog_x86.c

@@ -193,10 +193,7 @@ static void cmplog_handle_cmp_sub(GumCpuContext *context, gsize operand1,
 
   gsize address = ctx_read_reg(context, X86_REG_EIP);
 
-  register uintptr_t k = (uintptr_t)address;
-
-  k = (k >> 4) ^ (k << 8);
-  k &= CMP_MAP_W - 1;
+  register uintptr_t k = instrument_get_offset_hash(GUM_ADDRESS(address));
 
   if (__afl_cmp_map->headers[k].type != CMP_TYPE_INS)
     __afl_cmp_map->headers[k].hits = 0;

frida_mode/src/instrument/instrument_arm32.c

@@ -273,7 +273,19 @@ void instrument_flush(GumStalkerOutput *output) {
 
 gpointer instrument_cur(GumStalkerOutput *output) {
 
-  return gum_arm_writer_cur(output->writer.arm);
+  gpointer curr = NULL;
+
+  if (output->encoding == GUM_INSTRUCTION_SPECIAL) {
+
+    curr = gum_thumb_writer_cur(output->writer.thumb);
+
+  } else {
+
+    curr = gum_arm_writer_cur(output->writer.arm);
+
+  }
+
+  return curr;
 
 }
 

frida_mode/src/instrument/instrument_arm64.c

@@ -196,7 +196,7 @@ static void instrument_coverage_switch(GumStalkerObserver *self,
   insn = instrument_disassemble(from_insn);
   deterministic = instrument_is_deterministic(insn);
   cs_free(insn, 1);
-  if (deterministic) { return; }
+  if (!deterministic) { return; }
 
   /*
    * Since each block is prefixed with a restoration prologue, we need to be

frida_mode/test/png/GNUmakefile

@@ -7,10 +7,10 @@ LIBPNG_BUILD_DIR:=$(BUILD_DIR)libpng/
 HARNESS_BUILD_DIR:=$(BUILD_DIR)harness/
 PNGTEST_BUILD_DIR:=$(BUILD_DIR)pngtest/
 
-LIBZ_FILE:=$(LIBZ_BUILD_DIR)zlib-1.2.12.tar.gz
-LIBZ_URL:=http://www.zlib.net/zlib-1.2.12.tar.gz
-LIBZ_DIR:=$(LIBZ_BUILD_DIR)zlib-1.2.12/
-LIBZ_PC:=$(ZLIB_DIR)zlib.pc
+LIBZ_FILE:=$(LIBZ_BUILD_DIR)zlib-1.2.13.tar.gz
+LIBZ_URL:=http://www.zlib.net/zlib-1.2.13.tar.gz
+LIBZ_DIR:=$(LIBZ_BUILD_DIR)zlib-1.2.13/
+LIBZ_PC:=$(LIBZ_DIR)zlib.pc
 LIBZ_LIB:=$(LIBZ_DIR)libz.a
 
 LIBPNG_FILE:=$(LIBPNG_BUILD_DIR)libpng-1.2.56.tar.gz
@@ -48,7 +48,7 @@ all: $(TEST_BIN)
 	CFLAGS="-m32" LDFLAGS="-m32" make $(TEST_BIN)
 
 arm:
-	ARCH="arm" CC="arm-linux-gnueabihf-gcc" CXX="arm-linux-gnueabihf-g++" make $(TEST_BIN)
+	CFLAGS="-marm" LDFLAGS="-marm" CC="arm-linux-gnueabihf-gcc" CXX="arm-linux-gnueabihf-g++" make $(TEST_BIN)
 
 $(BUILD_DIR):
 	mkdir -p $@
@@ -96,7 +96,7 @@ $(LIBZ_PC): | $(LIBZ_DIR)
 			--static \
 			--archs="$(ARCH)"
 
-$(LIBZ_LIB): $(LIBZ_PC)
+$(LIBZ_LIB): | $(LIBZ_PC)
 	CFLAGS="$(CFLAGS) -fPIC" \
 		make \
 			-C $(LIBZ_DIR) \
@@ -133,7 +133,7 @@ png: $(LIBPNG_LIB)
 
 ######### TEST ########
 
-$(TEST_BIN): $(HARNESS_OBJ) $(PNGTEST_OBJ) $(LIBPNG_LIB)
+$(TEST_BIN): $(HARNESS_OBJ) $(PNGTEST_OBJ) $(LIBPNG_LIB) $(LIBZ_LIB)
 	$(CXX) \
 		$(CFLAGS) \
 		$(LDFLAGS) \

frida_mode/test/testinstr/GNUmakefile

@@ -18,6 +18,9 @@ all: $(TESTINSTBIN)
 32:
 	CFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all
 
+arm:
+	CFLAGS="-marm" LDFLAGS="-marm" CC="arm-linux-gnueabihf-gcc" CXX="arm-linux-gnueabihf-g++" make $(TESTINSTBIN)
+
 $(BUILD_DIR):
 	mkdir -p $@
 

include/afl-fuzz.h

@@ -169,12 +169,22 @@ struct queue_entry {
 
   u32 bitmap_size,                      /* Number of bits set in bitmap     */
       fuzz_level,                       /* Number of fuzzing iterations     */
-      n_fuzz_entry;                     /* offset in n_fuzz                 */
+      n_fuzz_entry                      /* offset in n_fuzz                 */
+#ifdef INTROSPECTION
+      ,
+      stats_selected,                   /* stats: how often selected        */
+      stats_skipped,                    /* stats: how often skipped         */
+      stats_finds,                      /* stats: # of saved finds          */
+      stats_crashes,                    /* stats: # of saved crashes        */
+      stats_tmouts                      /* stats: # of saved timeouts       */
+#endif
+      ;
 
   u64 exec_us,                          /* Execution time (us)              */
       handicap,                         /* Number of queue cycles behind    */
       depth,                            /* Path depth                       */
-      exec_cksum;                       /* Checksum of the execution trace  */
+      exec_cksum,                       /* Checksum of the execution trace  */
+      stats_mutated;                    /* stats: # of mutations performed  */
 
   u8 *trace_mini;                       /* Trace bytes, if kept             */
   u32 tc_ref;                           /* Trace bytes ref count            */
@@ -686,7 +696,8 @@ typedef struct afl_state {
   u32 plot_prev_qp, plot_prev_pf, plot_prev_pnf, plot_prev_ce, plot_prev_md;
   u64 plot_prev_qc, plot_prev_uc, plot_prev_uh, plot_prev_ed;
 
-  u64 stats_last_stats_ms, stats_last_plot_ms, stats_last_ms, stats_last_execs;
+  u64 stats_last_stats_ms, stats_last_plot_ms, stats_last_queue_ms,
+      stats_last_ms, stats_last_execs;
 
   /* StatsD */
   u64                statsd_last_send_ms;
@@ -1101,6 +1112,7 @@ void load_stats_file(afl_state_t *);
 void write_setup_file(afl_state_t *, u32, char **);
 void write_stats_file(afl_state_t *, u32, double, double, double);
 void maybe_update_plot_file(afl_state_t *, u32, double, double);
+void write_queue_stats(afl_state_t *);
 void show_stats(afl_state_t *);
 void show_stats_normal(afl_state_t *);
 void show_stats_pizza(afl_state_t *);

include/config.h

@@ -290,10 +290,11 @@
 
 #define UI_TARGET_HZ 5
 
-/* Fuzzer stats file and plot update intervals (sec): */
+/* Fuzzer stats file, queue stats and plot update intervals (sec): */
 
 #define STATS_UPDATE_SEC 60
 #define PLOT_UPDATE_SEC 5
+#define QUEUE_UPDATE_SEC 1800
 
 /* Smoothing divisor for CPU load and exec speed stats (1 - no smoothing). */
 

include/envs.h

@@ -124,7 +124,9 @@ static char *afl_environment_variables[] = {
     "AFL_LLVM_ALLOWLIST",
     "AFL_LLVM_DENYLIST",
     "AFL_LLVM_BLOCKLIST",
+    "AFL_CMPLOG",
     "AFL_LLVM_CMPLOG",
+    "AFL_GCC_CMPLOG",
     "AFL_LLVM_INSTRIM",
     "AFL_LLVM_CALLER",
     "AFL_LLVM_CTX",

qemu_mode/QEMUAFL_VERSION

@@ -1 +1 @@
-fa07ebfff5
+a8af9cbde7

src/afl-fuzz-init.c

@@ -1848,6 +1848,10 @@ static void handle_existing_out_dir(afl_state_t *afl) {
 
   }
 
+  fn = alloc_printf("%s/queue_data", afl->out_dir);
+  if (unlink(fn) && errno != ENOENT) { goto dir_cleanup_failed; }
+  ck_free(fn);
+
   fn = alloc_printf("%s/cmdline", afl->out_dir);
   if (unlink(fn) && errno != ENOENT) { goto dir_cleanup_failed; }
   ck_free(fn);