From 0b16533924c68001db8949247431685bac21060b Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Tue, 17 Dec 2019 09:57:29 +0100
Subject: [PATCH 1/4] Golang coverage report

---
 infra/base-images/base-builder/Dockerfile     |  1 +
 infra/base-images/base-builder/compile        |  2 +-
 .../base-builder/ossfuzz_coverage_runner.go   | 48 ++++++++++
 infra/base-images/base-runner/Dockerfile      | 17 ++++
 infra/base-images/base-runner/coverage        | 87 ++++++++++++-------
 infra/helper.py                               |  2 +-
 6 files changed, 123 insertions(+), 34 deletions(-)
 create mode 100644 infra/base-images/base-builder/ossfuzz_coverage_runner.go

diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile
index 6bedecbc6bab..e0d1c8ab246b 100644
--- a/infra/base-images/base-builder/Dockerfile
+++ b/infra/base-images/base-builder/Dockerfile
@@ -146,6 +146,7 @@ COPY compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
     precompile_honggfuzz srcmap write_labels.py /usr/local/bin/
 
 COPY detect_repo.py /opt/cifuzz/
+COPY ossfuzz_coverage_runner.go $GOPATH
 
 RUN precompile_honggfuzz
 
diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile
index 2a9876af2bdb..cdbbfe0a95a6 100755
--- a/infra/base-images/base-builder/compile
+++ b/infra/base-images/base-builder/compile
@@ -92,7 +92,7 @@ BUILD_CMD="bash -eux $SRC/build.sh"
 
 # We need to preserve source code files for generating a code coverage report.
 # We need exact files that were compiled, so copy both $SRC and $WORK dirs.
-COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $OUT"
+COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OUT"
 
 if [ "${BUILD_UID-0}" -ne "0" ]; then
   adduser -u $BUILD_UID --disabled-password --gecos '' builder
diff --git a/infra/base-images/base-builder/ossfuzz_coverage_runner.go b/infra/base-images/base-builder/ossfuzz_coverage_runner.go
new file mode 100644
index 000000000000..5519c6dc406d
--- /dev/null
+++ b/infra/base-images/base-builder/ossfuzz_coverage_runner.go
@@ -0,0 +1,48 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mypackagebeingfuzzed
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestFuzzCorpus(t *testing.T) {
+	dir := os.Getenv("FUZZ_CORPUS_DIR")
+	if dir == "" {
+		t.Logf("No fuzzing corpus directory set")
+		return
+	}
+	infos, err := ioutil.ReadDir(dir)
+	if err != nil {
+		t.Logf("Not fuzzing corpus directory %s", err)
+		return
+	}
+	filename := ""
+	defer func() {
+		if r := recover(); r != nil {
+			t.Error("Fuzz panicked in "+filename, r)
+		}
+	}()
+	for i := range infos {
+		filename = dir + infos[i].Name()
+		data, err := ioutil.ReadFile(filename)
+		if err != nil {
+			t.Error("Failed to read corpus file", err)
+		}
+		FuzzFunction(data)
+	}
+}
diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
index b1b7bec93043..f13d62beab92 100644
--- a/infra/base-images/base-runner/Dockerfile
+++ b/infra/base-images/base-runner/Dockerfile
@@ -24,6 +24,7 @@ COPY --from=base-clang /usr/local/bin/llvm-cov /usr/local/bin/
 COPY --from=base-clang /usr/local/bin/llvm-profdata /usr/local/bin/
 COPY --from=base-clang /usr/local/bin/llvm-symbolizer /usr/local/bin/
 
+RUN apt-get update
 RUN apt-get install -y \
     binutils \
     file \
@@ -64,3 +65,19 @@ ENV MSAN_OPTIONS="print_stats=1:strip_path_prefix=/workspace/:symbolize=1:dedup_
 ENV UBSAN_OPTIONS="print_stacktrace=1:print_summary=1:silence_unsigned_overflow=1:strip_path_prefix=/workspace/:symbolize=1:dedup_token_length=3"
 ENV FUZZER_ARGS="-rss_limit_mb=2560 -timeout=25"
 ENV AFL_FUZZER_ARGS="-m none"
+
+# Download and install the latest stable Go.
+ADD https://storage.googleapis.com/golang/getgo/installer_linux $SRC/
+RUN chmod +x $SRC/installer_linux && \
+    SHELL="bash" $SRC/installer_linux && \
+    rm $SRC/installer_linux
+
+# Set up Golang environment variables (copied from /root/.bash_profile).
+ENV GOPATH /root/go
+
+# /root/.go/bin is for the standard Go binaries (i.e. go, gofmt, etc).
+# $GOPATH/bin is for the binaries from the dependencies installed via "go get".
+ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
+
+# gocovmerge merges coverage profiles.
+RUN go get -u github.com/wadey/gocovmerge
diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage
index d2bd4ddd9ac2..93b8aee10c60 100755
--- a/infra/base-images/base-runner/coverage
+++ b/infra/base-images/base-runner/coverage
@@ -111,21 +111,35 @@ function run_fuzz_target {
   fi
 }
 
+export GOPATH=$OUT/$GOPATH
 # Run each fuzz target, generate raw coverage dumps.
 for fuzz_target in $FUZZ_TARGETS; do
-  # Continue if not a fuzz target.
-  if [[ $FUZZING_ENGINE != "none" ]]; then
-    grep "LLVMFuzzerTestOneInput" $fuzz_target > /dev/null 2>&1 || continue
-  fi
-
-  echo "Running $fuzz_target"
-  run_fuzz_target $fuzz_target &
-
-  if [[ -z $objects ]]; then
-    # The first object needs to be passed without -object= flag.
-    objects="$fuzz_target"
+  # Test if fuzz target is a golang one.
+  if [[ $FUZZING_LANGUAGE == "go" ]]; then
+    # Continue if not a fuzz target.
+    if [[ $FUZZING_ENGINE != "none" ]]; then
+      grep "go test -run" $fuzz_target > /dev/null 2>&1 || continue
+    fi
+    cd $GOPATH/src
+    echo "Running go target $fuzz_target"
+    export FUZZ_CORPUS_DIR="/corpus/${fuzz_target}/"
+    bash $OUT/$fuzz_target $DUMPS_DIR/$fuzz_target.profdata &
+    cd $OUT
   else
-    objects="$objects -object=$fuzz_target"
+    # Continue if not a fuzz target.
+    if [[ $FUZZING_ENGINE != "none" ]]; then
+      grep "LLVMFuzzerTestOneInput" $fuzz_target > /dev/null 2>&1 || continue
+    fi
+
+    echo "Running $fuzz_target"
+    run_fuzz_target $fuzz_target &
+
+    if [[ -z $objects ]]; then
+      # The first object needs to be passed without -object= flag.
+      objects="$fuzz_target"
+    else
+      objects="$objects -object=$fuzz_target"
+    fi
   fi
 
   # Do not spawn more processes than the number of CPUs available.
@@ -139,32 +153,41 @@ done
 # Wait for background processes to finish.
 wait
 
-# From this point on the script does not tolerate any errors.
-set -e
+if [[ $FUZZING_LANGUAGE == "go" ]]; then
+  /root/go/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov
+  go tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html
+  cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html
+  echo "Finished generating code coverage report for Go fuzz targets."
+else
 
-# Merge all dumps from the individual targets.
-rm -f $PROFILE_FILE
-llvm-profdata merge -sparse $DUMPS_DIR/*.profdata -o $PROFILE_FILE
+  # From this point on the script does not tolerate any errors.
+  set -e
 
-# TODO(mmoroz): add script from Chromium for rendering directory view reports.
-# The first path in $objects does not have -object= prefix (llvm-cov format).
-shared_libraries=$(coverage_helper shared_libs -build-dir=$OUT -object=$objects)
-objects="$objects $shared_libraries"
+  # Merge all dumps from the individual targets.
+  rm -f $PROFILE_FILE
+  llvm-profdata merge -sparse $DUMPS_DIR/*.profdata -o $PROFILE_FILE
 
-# It's important to use $LLVM_COV_COMMON_ARGS as the last argument due to
-# positional arguments (SOURCES) that can be passed via $COVERAGE_EXTRA_ARGS.
-LLVM_COV_ARGS="-instr-profile=$PROFILE_FILE $objects $LLVM_COV_COMMON_ARGS"
+  # TODO(mmoroz): add script from Chromium for rendering directory view reports.
+  # The first path in $objects does not have -object= prefix (llvm-cov format).
+  shared_libraries=$(coverage_helper shared_libs -build-dir=$OUT -object=$objects)
+  objects="$objects $shared_libraries"
 
-# Generate HTML report.
-llvm-cov show -format=html -output-dir=$REPORT_ROOT_DIR \
-    -Xdemangler c++filt -Xdemangler -n $LLVM_COV_ARGS
+  # It's important to use $LLVM_COV_COMMON_ARGS as the last argument due to
+  # positional arguments (SOURCES) that can be passed via $COVERAGE_EXTRA_ARGS.
+  LLVM_COV_ARGS="-instr-profile=$PROFILE_FILE $objects $LLVM_COV_COMMON_ARGS"
 
-# Export coverage summary in JSON format.
-llvm-cov export -summary-only $LLVM_COV_ARGS > $SUMMARY_FILE
+  # Generate HTML report.
+  llvm-cov show -format=html -output-dir=$REPORT_ROOT_DIR \
+      -Xdemangler c++filt -Xdemangler -n $LLVM_COV_ARGS
 
-# Post process HTML report.
-coverage_helper -v post_process -src-root-dir=/ -summary-file=$SUMMARY_FILE \
-    -output-dir=$REPORT_ROOT_DIR $PATH_EQUIVALENCE_ARGS
+  # Export coverage summary in JSON format.
+  llvm-cov export -summary-only $LLVM_COV_ARGS > $SUMMARY_FILE
+
+  # Post process HTML report.
+  coverage_helper -v post_process -src-root-dir=/ -summary-file=$SUMMARY_FILE \
+      -output-dir=$REPORT_ROOT_DIR $PATH_EQUIVALENCE_ARGS
+
+fi
 
 if [[ -n $HTTP_PORT ]]; then
   # Serve the report locally.
diff --git a/infra/helper.py b/infra/helper.py
index 99d87112989c..d91239df712a 100755
--- a/infra/helper.py
+++ b/infra/helper.py
@@ -59,7 +59,7 @@
 PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)')
 
 # Languages from project.yaml that have code coverage support.
-LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++']
+LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go']
 
 
 def main():  # pylint: disable=too-many-branches,too-many-return-statements,too-many-statements

From 5e806afb9174dd1c82ef7eb75fa50552e7fd49d2 Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Thu, 12 Nov 2020 13:58:35 +0100
Subject: [PATCH 2/4] Enables golang coverage report for gonids and go-dns

---
 projects/go-dns/build.sh   | 16 ++++++++++++++++
 projects/gonids/Dockerfile |  4 ++--
 projects/gonids/build.sh   | 18 +++++++++++++++++-
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/projects/go-dns/build.sh b/projects/go-dns/build.sh
index b9309993de7c..873a291d6e90 100755
--- a/projects/go-dns/build.sh
+++ b/projects/go-dns/build.sh
@@ -20,6 +20,22 @@ function compile_fuzzer {
   function=$2
   fuzzer=$3
 
+  if [[ $SANITIZER = *coverage* ]]; then
+    cd $GOPATH/src/$path
+    fuzzed_package=`pwd | rev | cut -d'/' -f 1 | rev`
+    cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go
+    sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go
+    sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
+    sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
+
+    echo "#/bin/sh" > $OUT/$fuzzer
+    echo "cd $path" >> $OUT/$fuzzer
+    echo "go test -run Test${function}Corpus -v -tags fuzz -coverprofile \$1 " >> $OUT/$fuzzer
+    chmod +x $OUT/$fuzzer
+
+    cd -
+    return 0
+  fi
   # Compile and instrument all Go files relevant to this fuzz target.
   go-fuzz -tags fuzz -func $function -o $fuzzer.a $path
 
diff --git a/projects/gonids/Dockerfile b/projects/gonids/Dockerfile
index a6231a16b717..66e9a8933c41 100644
--- a/projects/gonids/Dockerfile
+++ b/projects/gonids/Dockerfile
@@ -15,9 +15,9 @@
 ################################################################################
 
 FROM gcr.io/oss-fuzz-base/base-builder
-RUN go get github.com/google/gonids
+RUN go get -t github.com/google/gonids
 
 ADD https://rules.emergingthreats.net/open/suricata/emerging.rules.zip emerging.rules.zip
 
 COPY build.sh $SRC/
-WORKDIR $SRC
+WORKDIR $SRC/
diff --git a/projects/gonids/build.sh b/projects/gonids/build.sh
index 277b1c59c128..97fe7d5eef5d 100755
--- a/projects/gonids/build.sh
+++ b/projects/gonids/build.sh
@@ -20,6 +20,22 @@ function compile_fuzzer {
   function=$2
   fuzzer=$3
 
+  if [[ $SANITIZER = *coverage* ]]; then
+    cd $GOPATH/src/$path
+    fuzzed_package=`pwd | rev | cut -d'/' -f 1 | rev`
+    cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go
+    sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go
+    sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
+    sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
+
+    echo "#/bin/sh" > $OUT/$fuzzer
+    echo "cd $path" >> $OUT/$fuzzer
+    echo "go test -run Test${function}Corpus -v -coverprofile \$1 " >> $OUT/$fuzzer
+    chmod +x $OUT/$fuzzer
+
+    cd -
+    return 0
+  fi
   # Compile and instrument all Go files relevant to this fuzz target.
   go-fuzz -func $function -o $fuzzer.a $path
 
@@ -37,4 +53,4 @@ mkdir corpus
 set +x
 cat *.rules | while read l; do echo $l > corpus/$i.rule; i=$((i+1)); done
 set -x
-zip -r $OUT/fuzz_parserule_seed_corpus.zip corpus
+zip -q -r $OUT/fuzz_parserule_seed_corpus.zip corpus

From 053a37509c5e11020649d5796754c3039497c6f4 Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Thu, 12 Nov 2020 21:10:06 +0100
Subject: [PATCH 3/4] Generates summary for golang coverage reports

---
 infra/base-images/base-runner/Dockerfile | 3 +++
 infra/base-images/base-runner/coverage   | 4 +++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
index f13d62beab92..4e13fdcba4e2 100644
--- a/infra/base-images/base-runner/Dockerfile
+++ b/infra/base-images/base-runner/Dockerfile
@@ -81,3 +81,6 @@ ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
 
 # gocovmerge merges coverage profiles.
 RUN go get -u github.com/wadey/gocovmerge
+# gocovsum produces a json summary.
+RUN go get -u github.com/catenacyber/gocovsum
+
diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage
index 93b8aee10c60..8f8204654a81 100755
--- a/infra/base-images/base-runner/coverage
+++ b/infra/base-images/base-runner/coverage
@@ -111,6 +111,7 @@ function run_fuzz_target {
   fi
 }
 
+export SYSGOPATH=$GOPATH
 export GOPATH=$OUT/$GOPATH
 # Run each fuzz target, generate raw coverage dumps.
 for fuzz_target in $FUZZ_TARGETS; do
@@ -154,8 +155,9 @@ done
 wait
 
 if [[ $FUZZING_LANGUAGE == "go" ]]; then
-  /root/go/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov
+  $SYSGOPATH/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov
   go tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html
+  $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE
   cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html
   echo "Finished generating code coverage report for Go fuzz targets."
 else

From b897a1f71341b2f35115cba31f3755a593f27b45 Mon Sep 17 00:00:00 2001
From: Philippe Antoine <contact@catenacyber.fr>
Date: Thu, 12 Nov 2020 21:49:39 +0100
Subject: [PATCH 4/4] Performance profile for golang projects

---
 .../base-builder/ossfuzz_coverage_runner.go   | 21 +++++++++++++++++++
 infra/base-images/base-runner/Dockerfile      |  2 ++
 infra/base-images/base-runner/coverage        |  6 ++++++
 3 files changed, 29 insertions(+)

diff --git a/infra/base-images/base-builder/ossfuzz_coverage_runner.go b/infra/base-images/base-builder/ossfuzz_coverage_runner.go
index 5519c6dc406d..d433da24638c 100644
--- a/infra/base-images/base-builder/ossfuzz_coverage_runner.go
+++ b/infra/base-images/base-builder/ossfuzz_coverage_runner.go
@@ -17,6 +17,7 @@ package mypackagebeingfuzzed
 import (
 	"io/ioutil"
 	"os"
+	"runtime/pprof"
 	"testing"
 )
 
@@ -37,6 +38,15 @@ func TestFuzzCorpus(t *testing.T) {
 			t.Error("Fuzz panicked in "+filename, r)
 		}
 	}()
+	profname := os.Getenv("FUZZ_PROFILE_NAME")
+	if profname != "" {
+		f, err := os.Create(profname + ".cpu.prof")
+		if err != nil {
+			t.Logf("error creating profile file %s\n", err)
+		} else {
+			_ = pprof.StartCPUProfile(f)
+		}
+	}
 	for i := range infos {
 		filename = dir + infos[i].Name()
 		data, err := ioutil.ReadFile(filename)
@@ -45,4 +55,15 @@ func TestFuzzCorpus(t *testing.T) {
 		}
 		FuzzFunction(data)
 	}
+	if profname != "" {
+		pprof.StopCPUProfile()
+		f, err := os.Create(profname + ".heap.prof")
+		if err != nil {
+			t.Logf("error creating heap profile file %s\n", err)
+		}
+		if err = pprof.WriteHeapProfile(f); err != nil {
+			t.Logf("error writing heap profile file %s\n", err)
+		}
+		f.Close()
+	}
 }
diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
index 4e13fdcba4e2..85d5f6d6a4d9 100644
--- a/infra/base-images/base-runner/Dockerfile
+++ b/infra/base-images/base-runner/Dockerfile
@@ -81,6 +81,8 @@ ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
 
 # gocovmerge merges coverage profiles.
 RUN go get -u github.com/wadey/gocovmerge
+# pprof-merge merges performance profiles.
+RUN go get -u github.com/rakyll/pprof-merge
 # gocovsum produces a json summary.
 RUN go get -u github.com/catenacyber/gocovsum
 
diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage
index 8f8204654a81..f4f3161070ab 100755
--- a/infra/base-images/base-runner/coverage
+++ b/infra/base-images/base-runner/coverage
@@ -124,6 +124,7 @@ for fuzz_target in $FUZZ_TARGETS; do
     cd $GOPATH/src
     echo "Running go target $fuzz_target"
     export FUZZ_CORPUS_DIR="/corpus/${fuzz_target}/"
+    export FUZZ_PROFILE_NAME="$DUMPS_DIR/$fuzz_target.perf"
     bash $OUT/$fuzz_target $DUMPS_DIR/$fuzz_target.profdata &
     cd $OUT
   else
@@ -159,6 +160,11 @@ if [[ $FUZZING_LANGUAGE == "go" ]]; then
   go tool cover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html
   $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE
   cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html
+  $SYSGOPATH/bin/pprof-merge $DUMPS_DIR/*.perf.cpu.prof
+  mv merged.data $REPORT_ROOT_DIR/cpu.prof
+  $SYSGOPATH/bin/pprof-merge $DUMPS_DIR/*.perf.heap.prof
+  mv merged.data $REPORT_ROOT_DIR/heap.prof
+  #TODO some proxy for go tool pprof -http=127.0.0.1:8001 $DUMPS_DIR/cpu.prof
   echo "Finished generating code coverage report for Go fuzz targets."
 else