diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 7df6dc6b..a74869a5 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -6,9 +6,11 @@ name: fuzzing # to guard against code introducing crashes in the Markdown parsing, # which should in principle always run against any text input. # See: https://google.github.io/oss-fuzz/getting-started/continuous-integration/#how-it-works +# Note, to reproduce a crash locally, copy to `testcase` file` and run: `tox -e fuzz` on: pull_request: + paths-ignore: ['docs/**', 'tests/**'] jobs: Fuzzing: diff --git a/docs/conf.py b/docs/conf.py index 786eff04..52deea47 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -102,7 +102,7 @@ def run_apidoc(app): this_folder = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) api_folder = os.path.join(this_folder, "api") module_path = os.path.normpath(os.path.join(this_folder, "../")) - ignore_paths = ["../profiler.py", "../conftest.py", "../tests", "../benchmarking"] + ignore_paths = ["../scripts", "../conftest.py", "../tests", "../benchmarking"] ignore_paths = [ os.path.normpath(os.path.join(this_folder, p)) for p in ignore_paths ] diff --git a/scripts/build_fuzzers.py b/scripts/build_fuzzers.py new file mode 100644 index 00000000..3dce8ddf --- /dev/null +++ b/scripts/build_fuzzers.py @@ -0,0 +1,42 @@ +"""Build fuzzers idempotently in a given folder.""" +import argparse +from pathlib import Path +import subprocess + + +def main(): + """Build fuzzers idempotently in a given folder.""" + parser = argparse.ArgumentParser() + parser.add_argument("folder") + args = parser.parse_args() + folder = Path(args.folder) + if not folder.exists(): + print(f"Cloning google/oss-fuzz into: {folder}") + folder.mkdir(parents=True) + subprocess.check_call( + [ + "git", + "clone", + "--single-branch", + "https://github.com/google/oss-fuzz", + str(folder), + ] + ) + else: + print(f"Using google/oss-fuzz in: {folder}") + if not (folder / "build").exists(): + print(f"Building fuzzers in: {folder / 'build'}") + subprocess.check_call( + [ + "python", + str(folder / "infra" / "helper.py"), + "build_fuzzers", + "markdown-it-py", + ] + ) + else: + print(f"Using existing fuzzers in: {folder / 'build'}") + + +if __name__ == "__main__": + main() diff --git a/profiler.py b/scripts/profiler.py similarity index 100% rename from profiler.py rename to scripts/profiler.py diff --git a/tox.ini b/tox.ini index 85a7179b..bf0c8367 100644 --- a/tox.ini +++ b/tox.ini @@ -55,11 +55,18 @@ allowlist_externals = dot commands = mkdir -p "{toxworkdir}/prof" - python -m cProfile -o "{toxworkdir}/prof/output.pstats" profiler.py + python -m cProfile -o "{toxworkdir}/prof/output.pstats" scripts/profiler.py gprof2dot -f pstats -o "{toxworkdir}/prof/output.dot" "{toxworkdir}/prof/output.pstats" dot -Tsvg -o "{toxworkdir}/prof/output.svg" "{toxworkdir}/prof/output.dot" python -c 'import pathlib; print("profiler svg output under file://\{0\}".format(pathlib.Path(r"{toxworkdir}") / "prof" / "output.svg"))' +[testenv:fuzz] +description = run fuzzer on testcase file +; See: https://google.github.io/oss-fuzz/ +deps = atheris +commands_pre = python scripts/build_fuzzers.py {envdir}/oss-fuzz +commands = python {envdir}/oss-fuzz/infra/helper.py reproduce markdown-it-py fuzz_markdown {posargs:testcase} + [flake8] max-line-length = 100 extend-ignore = E203