Refactor infra of Go 1.18 fuzzers

docs/getting-started/new-project-guide/go_lang.md

@@ -109,7 +109,7 @@ For go-fuzz fuzzers, the best way to do this is by using the [`compile_go_fuzzer
 `compile_native_go_fuzzer` requires two dependencies which can be installed with:
 ```bash
 go install github.com/AdamKorcz/go-118-fuzz-build@latest
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+go get github.com/AdamKorcz/go-118-fuzz-build/testing
 ```
 
 A usage example from go-dns project is

infra/base-images/base-builder-go/Dockerfile

@@ -27,5 +27,4 @@ RUN install_go.sh
 
 # TODO(jonathanmetzman): Install this file using install_go.sh.
 COPY ossfuzz_coverage_runner.go \
-     native_ossfuzz_coverage_runner.go \
      $GOPATH/

infra/base-images/base-builder/compile_native_go_fuzzer

@@ -15,60 +15,25 @@
 #
 ################################################################################
 
-# Rewrites a copy of the fuzzer to allow for 
-# libFuzzer instrumentation.
-function rewrite_go_fuzz_harness() {
-	fuzzer_filename=$1
-	fuzz_function=$2
-
-	# Create a copy of the fuzzer to not modify the existing fuzzer.
-	cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go
-	mv $fuzzer_filename /tmp/
-	fuzzer_fn="${fuzzer_filename}"_fuzz_.go
-
-	# Replace *testing.F with *go118fuzzbuildutils.F.
-	echo "replacing *testing.F"
-	sed -i "s/func $fuzz_function(\([a-zA-Z0-9]*\) \*testing\.F)/func $fuzz_function(\1 \*go118fuzzbuildutils\.F)/g" "${fuzzer_fn}"
-
-	# Import https://github.com/AdamKorcz/go-118-fuzz-build.
-	# This changes the line numbers from the original fuzzer.
-	addimport -path "${fuzzer_fn}"
-	echo -e "\nvar _ = testing.Verbose // Ensure testing import remains\n" >>  "${fuzzer_fn}"
-}
-
 function build_native_go_fuzzer() {
 	fuzzer=$1
 	function=$2
 	path=$3
 	tags="-tags gofuzz"
 
-	if [[ $SANITIZER = *coverage* ]]; then
-		echo "here we perform coverage build"
-		fuzzed_package=`go list $tags -f '{{.Name}}' $path`
-		abspath=`go list $tags -f {{.Dir}} $path`
-		cd $abspath
-		cp $GOPATH/native_ossfuzz_coverage_runner.go ./"${function,,}"_test.go
-		sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go
-		sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
-		sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
-
-		# The repo is the module path/name, which is already created above
-		# in case it doesn't exist, but not always the same as the module
-		# path. This is necessary to handle SIV properly.
-		fuzzed_repo=$(go list $tags -f {{.Module}} "$path")
-		abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo`
-		# give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir
-		echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath
-		go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path
-
-		rm ./"${function,,}"_test.go
+	if [[ $SANITIZER == *coverage* ]]; then
+		current_dir=$(pwd)
+		mkdir $OUT/rawfuzzers || true
+		cd $abs_file_dir
+		go test -c -run $fuzzer -o $OUT/$fuzzer -cover
+		cp "${fuzzer_filename}" "${OUT}/rawfuzzers/${fuzzer}"
+		cd $current_dir
 	else
 		go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir
 		$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer
 	fi
 }
 
-
 path=$1
 function=$2
 fuzzer=$3
@@ -83,13 +48,7 @@ fuzzer_filename=$(grep -r -l --include='*.go' -s "$function" "${abs_file_dir}")
 # Test if file contains a line with "func $function" and "testing.F".
 if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ]
 then
-
-	rewrite_go_fuzz_harness $fuzzer_filename $function
 	build_native_go_fuzzer $fuzzer $function $abs_file_dir
-
-	# Clean up.
-	rm "${fuzzer_filename}_fuzz_.go"
-	mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename
 else
 	echo "Could not find the function: func ${function}(f *testing.F)"
 fi

infra/base-images/base-builder/install_go.sh

@@ -28,11 +28,7 @@ go install github.com/mdempsky/go114-fuzz-build@latest
 ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz
 
 cd /tmp
-git clone https://github.com/AdamKorcz/go-118-fuzz-build
+git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build
 cd go-118-fuzz-build
 go build
 mv go-118-fuzz-build $GOPATH/bin/
-
-cd addimport
-go build
-mv addimport $GOPATH/bin/

infra/base-images/base-runner/coverage

@@ -131,19 +131,31 @@ function run_go_fuzz_target {
   echo "Running go target $target"
   export FUZZ_CORPUS_DIR="$CORPUS_DIR/${target}/"
   export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf"
+
+  # setup for native go fuzzers
+  cd $OUT
+  mkdir -p "testdata/fuzz/${target}"
+  cp -r "${FUZZ_CORPUS_DIR}*" "testdata/fuzz/${target}/"
+
+  # rewrite libFuzzer corpus to Std Go corpus
+  $SYSGOPATH/bin/convertcorpus $target "testdata/fuzz/${target}"
+  cd -
 
   timeout $TIMEOUT $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log
   if (( $? != 0 )); then
     echo "Error occured while running $target:"
     cat $LOGS_DIR/$target.log
   fi
 
+  # cleanup after native go fuzzers
+  rm -r "${OUT}/testdata/fuzz/${target}"
+
   # The Go 1.18 fuzzers are renamed to "*_fuzz_.go" during "infra/helper.py build_fuzzers".
   # They are are therefore refered to as "*_fuzz_.go" in the profdata files.
   # Since the copies named "*_fuzz_.go" do not exist in the file tree during
   # the coverage build, we change the references in the .profdata files
   # to the original file names.
-  sed -i "s/_test.go_fuzz_.go/_test.go/g" $DUMPS_DIR/$target.profdata
+  #sed -i "s/_test.go_fuzz_.go/_test.go/g" $DUMPS_DIR/$target.profdata
   # translate from golangish paths to current absolute paths
   cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done
   # cf PATH_EQUIVALENCE_ARGS
@@ -239,7 +251,7 @@ for fuzz_target in $FUZZ_TARGETS; do
   if [[ $FUZZING_LANGUAGE == "go" ]]; then
     # Continue if not a fuzz target.
     if [[ $FUZZING_ENGINE != "none" ]]; then
-      grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || continue
+      grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || grep "testing\.T" $fuzz_target > /dev/null 2>&1 || continue
     fi
     run_go_fuzz_target $fuzz_target &
   elif [[ $FUZZING_LANGUAGE == "python" ]]; then
@@ -288,6 +300,7 @@ done
 wait
 
 if [[ $FUZZING_LANGUAGE == "go" ]]; then
+  echo $DUMPS_DIR
   $SYSGOPATH/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov
   gotoolcover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html
   $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE

infra/base-images/base-runner/gocoverage/convertcorpus/go.mod

@@ -0,0 +1,10 @@
+module oss-fuzz.com/gocoverage/convertcorpus
+
+go 1.19
+
+require github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221110144148-3ffc89b74f84
+
+require (
+	github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+)

infra/base-images/base-runner/gocoverage/convertcorpus/go.sum

@@ -0,0 +1,24 @@
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952 h1:cs1LC1MGKD1O4neR89Rc24t0u15Vs5ASfUQ2tLr/KbY=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952/go.mod h1:i9fr2JpcEcY/IHEvzCM3qXUZYOQHgR89dt4es1CgMhc=
+github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221110144148-3ffc89b74f84 h1:a0NR83n+t4XyUh32ifxu6XsmeLMKyOx5Lxub9IeBM7k=
+github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221110144148-3ffc89b74f84/go.mod h1:pXIs8t4wo19ehhsffZsAZxSQ+oPUF41iiDrUaIDWKFU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
+github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

infra/base-images/base-runner/gocoverage/convertcorpus/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/AdamKorcz/go-118-fuzz-build/coverage"
+)
+
+// reads all corpus files in a directory and converts
+// them from libFuzzer format to native Go format.
+func main() {
+	if len(os.Args) != 3 {
+		fmt.Println(os.Args)
+		log.Fatalf("need exactly two argument")
+	}
+	FUZZERNAME := os.Args[1]
+	CORPUS_PATH := os.Args[2]
+
+	filepath.Walk(CORPUS_PATH, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return nil
+		}
+		if !info.Mode().IsRegular() {
+			return nil
+		}
+		libFuzzerSeed, err := os.ReadFile(path)
+		if err != nil {
+			panic(err)
+		}
+		out := os.Getenv("OUT")
+		fuzzerContents, err := os.ReadFile(filepath.Join(out, "rawfuzzers", FUZZERNAME))
+		if err != nil {
+			panic(err)
+		}
+		goSeed := coverage.ConvertLibfuzzerSeedToGoSeed(fuzzerContents, libFuzzerSeed, FUZZERNAME)
+		err = os.Remove(path)
+		if err != nil {
+			panic(err)
+		}
+		f, err := os.Create(path)
+		if err != nil {
+			panic(err)
+		}
+		defer f.Close()
+		_, err = f.Write([]byte(goSeed))
+		if err != nil {
+			panic(err)
+		}
+		return nil
+	})
+}

infra/base-images/base-runner/install_go.sh

@@ -22,11 +22,12 @@ case $(uname -m) in
       # Download and install the latest stable Go.
       wget https://storage.googleapis.com/golang/getgo/installer_linux -O $SRC/installer_linux
       chmod +x $SRC/installer_linux
-      SHELL="bash" $SRC/installer_linux -version 1.18beta2
+      SHELL="bash" $SRC/installer_linux -version 1.19
       rm $SRC/installer_linux
       # Set up Golang coverage modules.
       printf $(find . -name gocoverage)
       cd $GOPATH/gocoverage && /root/.go/bin/go install ./...
+      cd convertcorpus && /root/.go/bin/go install .
       cd /root/.go/src/cmd/cover && /root/.go/bin/go build && mv cover $GOPATH/bin/gotoolcover
       ;;
     aarch64)

projects/golang/build.sh

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build
+
 export FUZZ_ROOT="github.com/dvyukov/go-fuzz-corpus"
 
 cd $SRC/text
@@ -179,34 +181,37 @@ zip $OUT/fuzz_elf_open_seed_corpus.zip ./testdata/*
 
 cd $SRC/go/src/image/png
 go mod init pngPackage
-go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build
+go get github.com/AdamKorcz/go-118-fuzz-build/testing
 compile_native_go_fuzzer pngPackage FuzzDecode fuzz_png_decode
 zip $OUT/fuzz_png_decode_seed_corpus.zip ./testdata/*.png
 
 cd $SRC/go/src/image/gif
 go mod init gifPackage
-go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build
+go get github.com/AdamKorcz/go-118-fuzz-build/testing
 compile_native_go_fuzzer gifPackage FuzzDecode fuzz_gif_decode
 zip $OUT/fuzz_gif_decode_seed_corpus.zip $SRC/go/src/image/testdata/*.gif
 
 cd $SRC/go/src/compress/gzip
 go mod init gzipPackage
 go mod tidy
 find . -name "*_test.go" ! -name 'fuzz_test.go' -type f -exec rm -f {} +
-go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build
+go get github.com/AdamKorcz/go-118-fuzz-build/testing
 compile_native_go_fuzzer gzipPackage FuzzReader fuzz_std_lib_gzip_reader
 zip $OUT/fuzz_std_lib_gzip_reader_seed_corpus.zip $SRC/go/src/compress/gzip/testdata/*
 
 cd $SRC/go/src/html
 go mod init htmlPackage
 go mod tidy
-go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes
-go get github.com/AdamKorcz/go-118-fuzz-build/utils
+go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build
+go get github.com/AdamKorcz/go-118-fuzz-build/testing
 compile_go_fuzzer htmlPackage Fuzz fuzz_html_escape_unescape
 
+# golangs build from source currently breaks.
+exit 0
+
 # Install latest Go from master branch and build fuzzers again
 cd $SRC
 rm -r go

projects/helm/Dockerfile

@@ -16,6 +16,6 @@
 
 FROM gcr.io/oss-fuzz-base/base-builder-go
 RUN git clone --depth 1 https://github.com/helm/helm
-RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing
+RUN git clone --depth 1 --branch=helmfixnative https://github.com/AdamKorcz/cncf-fuzzing
 COPY build.sh $SRC/
 WORKDIR $SRC/helm