From 0b770d9717905e42b3f77f61a2a59b7e54aed85f Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 13:12:45 +0000 Subject: [PATCH 01/12] compile_native_go_fuzzer: refactor Signed-off-by: AdamKorcz --- .../base-builder/compile_native_go_fuzzer | 32 +------------------ 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/infra/base-images/base-builder/compile_native_go_fuzzer b/infra/base-images/base-builder/compile_native_go_fuzzer index a7c58a61905d..e5f97c8bbdd6 100755 --- a/infra/base-images/base-builder/compile_native_go_fuzzer +++ b/infra/base-images/base-builder/compile_native_go_fuzzer @@ -15,27 +15,6 @@ # ################################################################################ -# Rewrites a copy of the fuzzer to allow for -# libFuzzer instrumentation. -function rewrite_go_fuzz_harness() { - fuzzer_filename=$1 - fuzz_function=$2 - - # Create a copy of the fuzzer to not modify the existing fuzzer. - cp $fuzzer_filename "${fuzzer_filename}"_fuzz_.go - mv $fuzzer_filename /tmp/ - fuzzer_fn="${fuzzer_filename}"_fuzz_.go - - # Replace *testing.F with *go118fuzzbuildutils.F. - echo "replacing *testing.F" - sed -i "s/func $fuzz_function(\([a-zA-Z0-9]*\) \*testing\.F)/func $fuzz_function(\1 \*go118fuzzbuildutils\.F)/g" "${fuzzer_fn}" - - # Import https://github.com/AdamKorcz/go-118-fuzz-build. - # This changes the line numbers from the original fuzzer. - addimport -path "${fuzzer_fn}" - echo -e "\nvar _ = testing.Verbose // Ensure testing import remains\n" >> "${fuzzer_fn}" -} - function build_native_go_fuzzer() { fuzzer=$1 function=$2 @@ -60,15 +39,12 @@ function build_native_go_fuzzer() { # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path - - rm ./"${function,,}"_test.go else go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer fi } - path=$1 function=$2 fuzzer=$3 @@ -83,13 +59,7 @@ fuzzer_filename=$(grep -r -l --include='*.go' -s "$function" "${abs_file_dir}") # Test if file contains a line with "func $function" and "testing.F". if [ $(grep -r "func $function" $fuzzer_filename | grep "testing.F" | wc -l) -eq 1 ] then - - rewrite_go_fuzz_harness $fuzzer_filename $function build_native_go_fuzzer $fuzzer $function $abs_file_dir - - # Clean up. - rm "${fuzzer_filename}_fuzz_.go" - mv /tmp/$(basename $fuzzer_filename) $fuzzer_filename else echo "Could not find the function: func ${function}(f *testing.F)" -fi +fi \ No newline at end of file From 0224d1a6d25b4e9db080eb3b489023a695fd618d Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 13:27:32 +0000 Subject: [PATCH 02/12] replace all usage of github.com/AdamKorcz/go-118-fuzz-build/utils with github.com/AdamKorcz/go-118-fuzz-build/testing Signed-off-by: AdamKorcz --- .../new-project-guide/go_lang.md | 2 +- .../native_ossfuzz_coverage_runner.go | 3 +-- infra/base-images/base-builder/install_go.sh | 6 +----- projects/golang/build.sh | 18 ++++++++++-------- projects/rekor/build.sh | 7 +++++-- projects/scorecard-web/build.sh | 6 ++++-- projects/sigstore/build.sh | 7 +++++-- 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/docs/getting-started/new-project-guide/go_lang.md b/docs/getting-started/new-project-guide/go_lang.md index 8169ea1c9fe4..291b1a908f11 100644 --- a/docs/getting-started/new-project-guide/go_lang.md +++ b/docs/getting-started/new-project-guide/go_lang.md @@ -109,7 +109,7 @@ For go-fuzz fuzzers, the best way to do this is by using the [`compile_go_fuzzer `compile_native_go_fuzzer` requires two dependencies which can be installed with: ```bash go install github.com/AdamKorcz/go-118-fuzz-build@latest -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go get github.com/AdamKorcz/go-118-fuzz-build/testing ``` A usage example from go-dns project is diff --git a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go index 1e26d8b23667..343b7e9844d7 100644 --- a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go +++ b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go @@ -18,8 +18,7 @@ import ( "io/ioutil" "os" "runtime/pprof" - "testing" - "github.com/AdamKorcz/go-118-fuzz-build/utils" + "github.com/AdamKorcz/go-118-fuzz-build/testing" ) func TestFuzzCorpus(t *testing.T) { diff --git a/infra/base-images/base-builder/install_go.sh b/infra/base-images/base-builder/install_go.sh index d38f5c3159d9..440ee5f6990c 100755 --- a/infra/base-images/base-builder/install_go.sh +++ b/infra/base-images/base-builder/install_go.sh @@ -28,11 +28,7 @@ go install github.com/mdempsky/go114-fuzz-build@latest ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz cd /tmp -git clone https://github.com/AdamKorcz/go-118-fuzz-build +git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build cd go-118-fuzz-build go build mv go-118-fuzz-build $GOPATH/bin/ - -cd addimport -go build -mv addimport $GOPATH/bin/ diff --git a/projects/golang/build.sh b/projects/golang/build.sh index 1ca2e1fea22a..93d42583e175 100755 --- a/projects/golang/build.sh +++ b/projects/golang/build.sh @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build + export FUZZ_ROOT="github.com/dvyukov/go-fuzz-corpus" cd $SRC/text @@ -179,15 +181,15 @@ zip $OUT/fuzz_elf_open_seed_corpus.zip ./testdata/* cd $SRC/go/src/image/png go mod init pngPackage -go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_native_go_fuzzer pngPackage FuzzDecode fuzz_png_decode zip $OUT/fuzz_png_decode_seed_corpus.zip ./testdata/*.png cd $SRC/go/src/image/gif go mod init gifPackage -go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_native_go_fuzzer gifPackage FuzzDecode fuzz_gif_decode zip $OUT/fuzz_gif_decode_seed_corpus.zip $SRC/go/src/image/testdata/*.gif @@ -195,16 +197,16 @@ cd $SRC/go/src/compress/gzip go mod init gzipPackage go mod tidy find . -name "*_test.go" ! -name 'fuzz_test.go' -type f -exec rm -f {} + -go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_native_go_fuzzer gzipPackage FuzzReader fuzz_std_lib_gzip_reader zip $OUT/fuzz_std_lib_gzip_reader_seed_corpus.zip $SRC/go/src/compress/gzip/testdata/* cd $SRC/go/src/html go mod init htmlPackage go mod tidy -go get github.com/AdamKorcz/go-118-fuzz-build/testingtypes -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_go_fuzzer htmlPackage Fuzz fuzz_html_escape_unescape # Install latest Go from master branch and build fuzzers again diff --git a/projects/rekor/build.sh b/projects/rekor/build.sh index 4d29a526c6c4..73a6acab50cf 100644 --- a/projects/rekor/build.sh +++ b/projects/rekor/build.sh @@ -15,8 +15,11 @@ # ################################################################################ -go install github.com/AdamKorcz/go-118-fuzz-build@latest -go get github.com/AdamKorcz/go-118-fuzz-build/utils +git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build + +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go install github.com/AdamKorcz/go-118-fuzz-build@dev +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_native_go_fuzzer github.com/sigstore/rekor/pkg/sharding FuzzCreateEntryIDFromParts FuzzCreateEntryIDFromParts compile_native_go_fuzzer github.com/sigstore/rekor/pkg/sharding FuzzGetUUIDFromIDString FuzzGetUUIDFromIDString diff --git a/projects/scorecard-web/build.sh b/projects/scorecard-web/build.sh index 2c0fc0fe7bbb..9712fbd047e7 100644 --- a/projects/scorecard-web/build.sh +++ b/projects/scorecard-web/build.sh @@ -15,9 +15,11 @@ # ################################################################################ +git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build -go install github.com/AdamKorcz/go-118-fuzz-build@latest -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go install github.com/AdamKorcz/go-118-fuzz-build@dev +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_native_go_fuzzer github.com/ossf/scorecard-webapp/app/server FuzzVerifyWorkflow FuzzVerifyWorkflow compile_native_go_fuzzer github.com/ossf/scorecard-webapp/app/server FuzzExtractCertInfo FuzzLoadCertificates diff --git a/projects/sigstore/build.sh b/projects/sigstore/build.sh index c6fc512f21a9..fbc89995dcee 100644 --- a/projects/sigstore/build.sh +++ b/projects/sigstore/build.sh @@ -15,13 +15,16 @@ # ################################################################################ +git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build + zip "${OUT}/FuzzLoadCertificates_seed_corpus.zip" corpus/pem/* zip "${OUT}/FuzzUnmarshalCertificatesFromPEM_seed_corpus.zip" corpus/pem/* zip "${OUT}/FuzzUnmarshalPEMToPublicKey_seed_corpus.zip" corpus/pem/* zip "${OUT}/FuzzED25529SignerVerfier_seed_corpus.zip" corpus/ed25519/* -go install github.com/AdamKorcz/go-118-fuzz-build@latest -go get github.com/AdamKorcz/go-118-fuzz-build/utils +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +go install github.com/AdamKorcz/go-118-fuzz-build@dev +go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_native_go_fuzzer github.com/sigstore/sigstore/test/fuzz FuzzGetPassword FuzzGetPassword compile_native_go_fuzzer github.com/sigstore/sigstore/test/fuzz/pem FuzzLoadCertificates FuzzLoadCertificates From 53e3f2249e5a25048c9df1323db8ca2e4589755d Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 17:55:27 +0000 Subject: [PATCH 03/12] rb --- projects/golang/build.sh | 5 ++++- projects/rekor/build.sh | 2 +- projects/scorecard-web/build.sh | 2 +- projects/sigstore/build.sh | 9 ++++----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/projects/golang/build.sh b/projects/golang/build.sh index 93d42583e175..77fa24698849 100755 --- a/projects/golang/build.sh +++ b/projects/golang/build.sh @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build +git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build export FUZZ_ROOT="github.com/dvyukov/go-fuzz-corpus" @@ -209,6 +209,9 @@ go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-bui go get github.com/AdamKorcz/go-118-fuzz-build/testing compile_go_fuzzer htmlPackage Fuzz fuzz_html_escape_unescape +# golangs build from source currently breaks. +exit 0 + # Install latest Go from master branch and build fuzzers again cd $SRC rm -r go diff --git a/projects/rekor/build.sh b/projects/rekor/build.sh index 73a6acab50cf..ffa3bae375d4 100644 --- a/projects/rekor/build.sh +++ b/projects/rekor/build.sh @@ -15,7 +15,7 @@ # ################################################################################ -git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build +git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build go install github.com/AdamKorcz/go-118-fuzz-build@dev diff --git a/projects/scorecard-web/build.sh b/projects/scorecard-web/build.sh index 9712fbd047e7..55d8257d457c 100644 --- a/projects/scorecard-web/build.sh +++ b/projects/scorecard-web/build.sh @@ -15,7 +15,7 @@ # ################################################################################ -git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build +git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build go install github.com/AdamKorcz/go-118-fuzz-build@dev diff --git a/projects/sigstore/build.sh b/projects/sigstore/build.sh index fbc89995dcee..abca981c5910 100644 --- a/projects/sigstore/build.sh +++ b/projects/sigstore/build.sh @@ -15,12 +15,11 @@ # ################################################################################ -git clone https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build +git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build -zip "${OUT}/FuzzLoadCertificates_seed_corpus.zip" corpus/pem/* -zip "${OUT}/FuzzUnmarshalCertificatesFromPEM_seed_corpus.zip" corpus/pem/* -zip "${OUT}/FuzzUnmarshalPEMToPublicKey_seed_corpus.zip" corpus/pem/* -zip "${OUT}/FuzzED25529SignerVerfier_seed_corpus.zip" corpus/ed25519/* +rm go.mod +rm go.sum +cd $SRC/sigstore go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build go install github.com/AdamKorcz/go-118-fuzz-build@dev From ad386f7fc42922a2bd92bb139d373c8a1f45fb62 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 18:04:20 +0000 Subject: [PATCH 04/12] rb --- projects/sigstore/build.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/sigstore/build.sh b/projects/sigstore/build.sh index abca981c5910..be253aeeca8b 100644 --- a/projects/sigstore/build.sh +++ b/projects/sigstore/build.sh @@ -17,6 +17,11 @@ git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build +zip "${OUT}/FuzzLoadCertificates_seed_corpus.zip" corpus/pem/* +zip "${OUT}/FuzzUnmarshalCertificatesFromPEM_seed_corpus.zip" corpus/pem/* +zip "${OUT}/FuzzUnmarshalPEMToPublicKey_seed_corpus.zip" corpus/pem/* +zip "${OUT}/FuzzED25529SignerVerfier_seed_corpus.zip" corpus/ed25519/* + rm go.mod rm go.sum cd $SRC/sigstore From 087afb9abde787df873b93197f114005ec5499a6 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 19:30:38 +0000 Subject: [PATCH 05/12] fix coverage build Signed-off-by: AdamKorcz --- .../base-builder-go/native_ossfuzz_coverage_runner.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go index 343b7e9844d7..fb2a20055ef5 100644 --- a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go +++ b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go @@ -18,7 +18,8 @@ import ( "io/ioutil" "os" "runtime/pprof" - "github.com/AdamKorcz/go-118-fuzz-build/testing" + "testing" + aTesting "github.com/AdamKorcz/go-118-fuzz-build/testing" ) func TestFuzzCorpus(t *testing.T) { @@ -53,7 +54,7 @@ func TestFuzzCorpus(t *testing.T) { if err != nil { t.Error("Failed to read corpus file", err) } - fuzzerF := &utils.F{Data:data, T:&testing.T{}} + fuzzerF := &aTesting.F{Data:data, T:&aTesting.T{}} FuzzFunction(fuzzerF) } if profname != "" { From a8c3b8045bfd4af990f4efd7989ad44473f38fe4 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 20:41:48 +0000 Subject: [PATCH 06/12] disable native go coverage build Signed-off-by: AdamKorcz --- .../base-builder/compile_native_go_fuzzer | 23 ++++--------------- 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/infra/base-images/base-builder/compile_native_go_fuzzer b/infra/base-images/base-builder/compile_native_go_fuzzer index e5f97c8bbdd6..7136c706260c 100755 --- a/infra/base-images/base-builder/compile_native_go_fuzzer +++ b/infra/base-images/base-builder/compile_native_go_fuzzer @@ -22,23 +22,10 @@ function build_native_go_fuzzer() { tags="-tags gofuzz" if [[ $SANITIZER = *coverage* ]]; then - echo "here we perform coverage build" - fuzzed_package=`go list $tags -f '{{.Name}}' $path` - abspath=`go list $tags -f {{.Dir}} $path` - cd $abspath - cp $GOPATH/native_ossfuzz_coverage_runner.go ./"${function,,}"_test.go - sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go - sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go - sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go - - # The repo is the module path/name, which is already created above - # in case it doesn't exist, but not always the same as the module - # path. This is necessary to handle SIV properly. - fuzzed_repo=$(go list $tags -f {{.Module}} "$path") - abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` - # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir - echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath - go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path + # TODO(AdamKorcz): rewrite the coverage part + # to a more native-Go-friendly implementation. + # The seed should be passed directly to the naitve + # go fuzzer instead of using a wrapper. else go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer @@ -62,4 +49,4 @@ then build_native_go_fuzzer $fuzzer $function $abs_file_dir else echo "Could not find the function: func ${function}(f *testing.F)" -fi \ No newline at end of file +fi From bffa7a710c214681d45a84e32692f3d0dfb2624b Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 7 Nov 2022 21:30:17 +0000 Subject: [PATCH 07/12] nit Signed-off-by: AdamKorcz --- infra/base-images/base-builder/compile_native_go_fuzzer | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/infra/base-images/base-builder/compile_native_go_fuzzer b/infra/base-images/base-builder/compile_native_go_fuzzer index 7136c706260c..331d3cbdf6db 100755 --- a/infra/base-images/base-builder/compile_native_go_fuzzer +++ b/infra/base-images/base-builder/compile_native_go_fuzzer @@ -21,12 +21,11 @@ function build_native_go_fuzzer() { path=$3 tags="-tags gofuzz" - if [[ $SANITIZER = *coverage* ]]; then - # TODO(AdamKorcz): rewrite the coverage part + if [[ $SANITIZER != *coverage* ]]; then + # TODO(AdamKorcz): rewrite the coverage build # to a more native-Go-friendly implementation. # The seed should be passed directly to the naitve # go fuzzer instead of using a wrapper. - else go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer fi From a15c74f7bdc624c1ad9bb43d211258ee8fe0d76b Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Thu, 10 Nov 2022 16:20:35 +0000 Subject: [PATCH 08/12] Add tool to convert libFuzzer seed to Native Go seed Signed-off-by: AdamKorcz --- .../base-builder/compile_native_go_fuzzer | 13 +++-- infra/base-images/base-runner/coverage | 19 ++++++- .../gocoverage/convertcorpus/go.mod | 10 ++++ .../gocoverage/convertcorpus/go.sum | 24 +++++++++ .../gocoverage/convertcorpus/main.go | 54 +++++++++++++++++++ infra/base-images/base-runner/install_go.sh | 3 +- 6 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 infra/base-images/base-runner/gocoverage/convertcorpus/go.mod create mode 100644 infra/base-images/base-runner/gocoverage/convertcorpus/go.sum create mode 100644 infra/base-images/base-runner/gocoverage/convertcorpus/main.go diff --git a/infra/base-images/base-builder/compile_native_go_fuzzer b/infra/base-images/base-builder/compile_native_go_fuzzer index 331d3cbdf6db..97d3a874477d 100755 --- a/infra/base-images/base-builder/compile_native_go_fuzzer +++ b/infra/base-images/base-builder/compile_native_go_fuzzer @@ -21,11 +21,14 @@ function build_native_go_fuzzer() { path=$3 tags="-tags gofuzz" - if [[ $SANITIZER != *coverage* ]]; then - # TODO(AdamKorcz): rewrite the coverage build - # to a more native-Go-friendly implementation. - # The seed should be passed directly to the naitve - # go fuzzer instead of using a wrapper. + if [[ $SANITIZER == *coverage* ]]; then + current_dir=$(pwd) + mkdir $OUT/rawfuzzers || true + cd $abs_file_dir + go test -c -run $fuzzer -o $OUT/$fuzzer -cover + cp "${fuzzer_filename}" "${OUT}/rawfuzzers/${fuzzer}" + cd $current_dir + else go-118-fuzz-build -o $fuzzer.a -func $function $abs_file_dir $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer fi diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 373a1e1b921c..c11e29735ffe 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -131,6 +131,15 @@ function run_go_fuzz_target { echo "Running go target $target" export FUZZ_CORPUS_DIR="$CORPUS_DIR/${target}/" export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf" + + # setup for native go fuzzers + cd $OUT + mkdir -p "testdata/fuzz/${target}" + cp -r "${FUZZ_CORPUS_DIR}*" "testdata/fuzz/${target}/" + + # rewrite libFuzzer corpus to Std Go corpus + $SYSGOPATH/bin/convertcorpus $target "testdata/fuzz/${target}" + cd - timeout $TIMEOUT $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log if (( $? != 0 )); then @@ -138,12 +147,15 @@ function run_go_fuzz_target { cat $LOGS_DIR/$target.log fi + # cleanup after native go fuzzers + rm -r "${OUT}/testdata/fuzz/${target}" + # The Go 1.18 fuzzers are renamed to "*_fuzz_.go" during "infra/helper.py build_fuzzers". # They are are therefore refered to as "*_fuzz_.go" in the profdata files. # Since the copies named "*_fuzz_.go" do not exist in the file tree during # the coverage build, we change the references in the .profdata files # to the original file names. - sed -i "s/_test.go_fuzz_.go/_test.go/g" $DUMPS_DIR/$target.profdata + #sed -i "s/_test.go_fuzz_.go/_test.go/g" $DUMPS_DIR/$target.profdata # translate from golangish paths to current absolute paths cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done # cf PATH_EQUIVALENCE_ARGS @@ -239,7 +251,7 @@ for fuzz_target in $FUZZ_TARGETS; do if [[ $FUZZING_LANGUAGE == "go" ]]; then # Continue if not a fuzz target. if [[ $FUZZING_ENGINE != "none" ]]; then - grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || continue + grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || grep "testing\.T" $fuzz_target > /dev/null 2>&1 || continue fi run_go_fuzz_target $fuzz_target & elif [[ $FUZZING_LANGUAGE == "python" ]]; then @@ -288,7 +300,10 @@ done wait if [[ $FUZZING_LANGUAGE == "go" ]]; then + ls $DUMPS_DIR + echo $DUMPS_DIR $SYSGOPATH/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov + #TODO(AdamKorcz): Get rid of this sed command gotoolcover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html diff --git a/infra/base-images/base-runner/gocoverage/convertcorpus/go.mod b/infra/base-images/base-runner/gocoverage/convertcorpus/go.mod new file mode 100644 index 000000000000..c82e4565f29a --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/convertcorpus/go.mod @@ -0,0 +1,10 @@ +module oss-fuzz.com/gocoverage/convertcorpus + +go 1.19 + +require github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221110144148-3ffc89b74f84 + +require ( + github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952 // indirect + github.com/cyphar/filepath-securejoin v0.2.3 // indirect +) diff --git a/infra/base-images/base-runner/gocoverage/convertcorpus/go.sum b/infra/base-images/base-runner/gocoverage/convertcorpus/go.sum new file mode 100644 index 000000000000..7b7d20e20253 --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/convertcorpus/go.sum @@ -0,0 +1,24 @@ +github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952 h1:cs1LC1MGKD1O4neR89Rc24t0u15Vs5ASfUQ2tLr/KbY= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20220824214621-3c06a36a6952/go.mod h1:i9fr2JpcEcY/IHEvzCM3qXUZYOQHgR89dt4es1CgMhc= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221110144148-3ffc89b74f84 h1:a0NR83n+t4XyUh32ifxu6XsmeLMKyOx5Lxub9IeBM7k= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221110144148-3ffc89b74f84/go.mod h1:pXIs8t4wo19ehhsffZsAZxSQ+oPUF41iiDrUaIDWKFU= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI= +github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/base-images/base-runner/gocoverage/convertcorpus/main.go b/infra/base-images/base-runner/gocoverage/convertcorpus/main.go new file mode 100644 index 000000000000..43e2c7ea7dd0 --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/convertcorpus/main.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/AdamKorcz/go-118-fuzz-build/coverage" +) + +// reads all corpus files in a directory and converts +// them from libFuzzer format to native Go format. +func main() { + if len(os.Args) != 3 { + fmt.Println(os.Args) + log.Fatalf("need exactly two argument") + } + FUZZERNAME := os.Args[1] + CORPUS_PATH := os.Args[2] + + filepath.Walk(CORPUS_PATH, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if !info.Mode().IsRegular() { + return nil + } + libFuzzerSeed, err := os.ReadFile(path) + if err != nil { + panic(err) + } + out := os.Getenv("OUT") + fuzzerContents, err := os.ReadFile(filepath.Join(out, "rawfuzzers", FUZZERNAME)) + if err != nil { + panic(err) + } + goSeed := coverage.ConvertLibfuzzerSeedToGoSeed(fuzzerContents, libFuzzerSeed, FUZZERNAME) + err = os.Remove(path) + if err != nil { + panic(err) + } + f, err := os.Create(path) + if err != nil { + panic(err) + } + defer f.Close() + _, err = f.Write([]byte(goSeed)) + if err != nil { + panic(err) + } + return nil + }) +} diff --git a/infra/base-images/base-runner/install_go.sh b/infra/base-images/base-runner/install_go.sh index 91e1effc552d..8b159b2385eb 100755 --- a/infra/base-images/base-runner/install_go.sh +++ b/infra/base-images/base-runner/install_go.sh @@ -22,11 +22,12 @@ case $(uname -m) in # Download and install the latest stable Go. wget https://storage.googleapis.com/golang/getgo/installer_linux -O $SRC/installer_linux chmod +x $SRC/installer_linux - SHELL="bash" $SRC/installer_linux -version 1.18beta2 + SHELL="bash" $SRC/installer_linux -version 1.19 rm $SRC/installer_linux # Set up Golang coverage modules. printf $(find . -name gocoverage) cd $GOPATH/gocoverage && /root/.go/bin/go install ./... + cd convertcorpus && /root/.go/bin/go install . cd /root/.go/src/cmd/cover && /root/.go/bin/go build && mv cover $GOPATH/bin/gotoolcover ;; aarch64) From 4f071c248975500ee6538d70ef368e9ef1795180 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Thu, 10 Nov 2022 20:21:49 +0000 Subject: [PATCH 09/12] remove native go test wrapper Signed-off-by: AdamKorcz --- infra/base-images/base-builder-go/Dockerfile | 1 - .../native_ossfuzz_coverage_runner.go | 71 ------------------- 2 files changed, 72 deletions(-) delete mode 100644 infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go diff --git a/infra/base-images/base-builder-go/Dockerfile b/infra/base-images/base-builder-go/Dockerfile index 5b4e01bb54ec..168d509023cf 100644 --- a/infra/base-images/base-builder-go/Dockerfile +++ b/infra/base-images/base-builder-go/Dockerfile @@ -27,5 +27,4 @@ RUN install_go.sh # TODO(jonathanmetzman): Install this file using install_go.sh. COPY ossfuzz_coverage_runner.go \ - native_ossfuzz_coverage_runner.go \ $GOPATH/ diff --git a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go b/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go deleted file mode 100644 index fb2a20055ef5..000000000000 --- a/infra/base-images/base-builder-go/native_ossfuzz_coverage_runner.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mypackagebeingfuzzed - -import ( - "io/ioutil" - "os" - "runtime/pprof" - "testing" - aTesting "github.com/AdamKorcz/go-118-fuzz-build/testing" -) - -func TestFuzzCorpus(t *testing.T) { - dir := os.Getenv("FUZZ_CORPUS_DIR") - if dir == "" { - t.Logf("No fuzzing corpus directory set") - return - } - infos, err := ioutil.ReadDir(dir) - if err != nil { - t.Logf("Not fuzzing corpus directory %s", err) - return - } - filename := "" - defer func() { - if r := recover(); r != nil { - t.Error("Fuzz panicked in "+filename, r) - } - }() - profname := os.Getenv("FUZZ_PROFILE_NAME") - if profname != "" { - f, err := os.Create(profname + ".cpu.prof") - if err != nil { - t.Logf("error creating profile file %s\n", err) - } else { - _ = pprof.StartCPUProfile(f) - } - } - for i := range infos { - filename = dir + infos[i].Name() - data, err := ioutil.ReadFile(filename) - if err != nil { - t.Error("Failed to read corpus file", err) - } - fuzzerF := &aTesting.F{Data:data, T:&aTesting.T{}} - FuzzFunction(fuzzerF) - } - if profname != "" { - pprof.StopCPUProfile() - f, err := os.Create(profname + ".heap.prof") - if err != nil { - t.Logf("error creating heap profile file %s\n", err) - } - if err = pprof.WriteHeapProfile(f); err != nil { - t.Logf("error writing heap profile file %s\n", err) - } - f.Close() - } -} From de6a585dfb0c4269c17cd12004f1f5a26976c82f Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Mon, 14 Nov 2022 09:52:12 +0000 Subject: [PATCH 10/12] remove debugging Signed-off-by: AdamKorcz --- infra/base-images/base-runner/coverage | 2 -- 1 file changed, 2 deletions(-) diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index c11e29735ffe..1d1e1d94bfea 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -300,10 +300,8 @@ done wait if [[ $FUZZING_LANGUAGE == "go" ]]; then - ls $DUMPS_DIR echo $DUMPS_DIR $SYSGOPATH/bin/gocovmerge $DUMPS_DIR/*.profdata > fuzz.cov - #TODO(AdamKorcz): Get rid of this sed command gotoolcover -html=fuzz.cov -o $REPORT_ROOT_DIR/index.html $SYSGOPATH/bin/gocovsum fuzz.cov > $SUMMARY_FILE cp $REPORT_ROOT_DIR/index.html $REPORT_PLATFORM_DIR/index.html From 37ab1dc0e76383bf4449d8bc0312e02a79f0bbe6 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Tue, 15 Nov 2022 11:39:14 +0000 Subject: [PATCH 11/12] Fix istio and helm Signed-off-by: AdamKorcz --- projects/helm/Dockerfile | 2 +- projects/istio/build.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/projects/helm/Dockerfile b/projects/helm/Dockerfile index 3f6327289827..6b8dbc70f32c 100644 --- a/projects/helm/Dockerfile +++ b/projects/helm/Dockerfile @@ -16,6 +16,6 @@ FROM gcr.io/oss-fuzz-base/base-builder-go RUN git clone --depth 1 https://github.com/helm/helm -RUN git clone --depth 1 https://github.com/cncf/cncf-fuzzing +RUN git clone --depth 1 --branch=helmfixnative https://github.com/AdamKorcz/cncf-fuzzing COPY build.sh $SRC/ WORKDIR $SRC/helm diff --git a/projects/istio/build.sh b/projects/istio/build.sh index 57f425ff6590..62f87c64670a 100644 --- a/projects/istio/build.sh +++ b/projects/istio/build.sh @@ -15,6 +15,15 @@ # ################################################################################ +cd $SRC +git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build +cd $SRC/istio +go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build +find $SRC/istio -type f -exec sed -i 's/AdamKorcz\/go-118-fuzz-build\/utils/AdamKorcz\/go-118-fuzz-build\/testing/g' {} \; +sed -i 's/\"testing\"/\"github.com\/AdamKorcz\/go-118-fuzz-build\/testing\"/g' $SRC/istio/pkg/fuzz/util.go +cat $SRC/istio/pkg/fuzz/util.go + + if [ -n "${OSS_FUZZ_CI-}" ] then echo "Skipping most fuzzers since the OSS-fuzz CI may fail from running out of disk space." From 8ff11571ad0abaa40d006e4a3d32a5d9a8c428d2 Mon Sep 17 00:00:00 2001 From: AdamKorcz Date: Wed, 16 Nov 2022 14:45:29 +0000 Subject: [PATCH 12/12] reinstate existing istio build.sh Signed-off-by: AdamKorcz --- projects/istio/build.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/projects/istio/build.sh b/projects/istio/build.sh index 62f87c64670a..57f425ff6590 100644 --- a/projects/istio/build.sh +++ b/projects/istio/build.sh @@ -15,15 +15,6 @@ # ################################################################################ -cd $SRC -git clone --branch=dev https://github.com/AdamKorcz/go-118-fuzz-build $SRC/go-118-fuzz-build -cd $SRC/istio -go mod edit -replace github.com/AdamKorcz/go-118-fuzz-build=/src/go-118-fuzz-build -find $SRC/istio -type f -exec sed -i 's/AdamKorcz\/go-118-fuzz-build\/utils/AdamKorcz\/go-118-fuzz-build\/testing/g' {} \; -sed -i 's/\"testing\"/\"github.com\/AdamKorcz\/go-118-fuzz-build\/testing\"/g' $SRC/istio/pkg/fuzz/util.go -cat $SRC/istio/pkg/fuzz/util.go - - if [ -n "${OSS_FUZZ_CI-}" ] then echo "Skipping most fuzzers since the OSS-fuzz CI may fail from running out of disk space."