Update fuzzing (openvinotoolkit#3829)

* update-fuzzing-tests

* extend available torch version

pyproject.toml

@@ -156,18 +156,16 @@ include = ["otx*"]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # COVERAGE CONFIGURATION.                                                     #
-[tool.coverage.paths]
-source = [
-    "src",
-]
-
 [tool.coverage.report]
 exclude_lines = [
     "pragma: no cover",
     "if TYPE_CHECKING:",
 ]
 
 [tool.coverage.run]
+source = [
+    "src/otx/",
+]
 omit = [
     "**/__init__.py",
     "src/otx/recipes/*",
@@ -184,6 +182,10 @@ omit = [
     "src/otx/core/data/transform_libs/mmseg.py",
     "src/otx/core/exporter/mmdeploy.py",
     "src/otx/core/model/utils/*",
+
+    # Ignore some generated files by opencv-python
+    "config.py",
+    "config-3.py",
 ]
 
 

src/otx/cli/utils/installation.py

@@ -25,6 +25,13 @@
     "2.0.0": {"torchvision": "0.15.1", "cuda": ("11.7", "11.8")},
     "2.0.1": {"torchvision": "0.15.2", "cuda": ("11.7", "11.8")},
     "2.1.1": {"torchvision": "0.16.1", "cuda": ("11.8", "12.1")},
+    "2.1.2": {"torchvision": "0.16.2", "cuda": ("11.8", "12.1")},
+    "2.2.0": {"torchvision": "0.17.0", "cuda": ("11.8", "12.1")},
+    "2.2.1": {"torchvision": "0.17.1", "cuda": ("11.8", "12.1")},
+    "2.2.2": {"torchvision": "0.17.2", "cuda": ("11.8", "12.1")},
+    "2.3.0": {"torchvision": "0.18.0", "cuda": ("11.8", "12.1")},
+    "2.3.1": {"torchvision": "0.18.1", "cuda": ("11.8", "12.1")},
+    "2.4.0": {"torchvision": "0.19.0", "cuda": ("11.8", "12.1", "12.4")},
 }
 
 MM_REQUIREMENTS = [

tests/fuzzing/assets/cli/commands.dict

@@ -1,3 +1,5 @@
+"-h"
+"-v"
 "install"
 "find"
 "train"
@@ -6,3 +8,4 @@
 "export"
 "optimize"
 "explain"
+"benchmark"

tests/fuzzing/cli_fuzzing.py

@@ -1,3 +1,5 @@
+#!.tox/fuzzing/bin/python
+
 import sys
 
 import atheris
@@ -17,8 +19,8 @@ def fuzz_otx(input_bytes):
     try:
         _ = cli_main()
     except SystemExit as e:
-        # argparser will throw SystemExit with code 2 when some required arguments are missing
-        if e.code != 2:
+        # argparser will throw SystemExit with code 0 or 2 when completed successfuly or some required arguments are missing
+        if e.code not in [0, 2]:
             raise
     finally:
         sys.argv = backup_argv

tests/fuzzing/eval_fuzzing_crash.py

@@ -0,0 +1,26 @@
+#!.tox/fuzzing/bin/python
+
+import sys
+from pathlib import Path
+
+from otx.cli import main as cli_main
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("usage: python eval_fuzzing_crash.py <fuzzing-crash-binary-file>")
+        return
+
+    ba = None
+    if Path.exists(sys.argv[1]):
+        with Path.open(sys.argv[1], "rb") as f:
+            ba = bytearray(f.read())
+
+    arguments = "".join(ba.decode(errors="replace"))
+
+    sys.argv = ["otx", f"{arguments!s}"]
+    _ = cli_main()
+
+
+if __name__ == "__main__":
+    main()

tox.ini

@@ -105,7 +105,13 @@ deps =
     atheris
     coverage
 extras = full
+commands_pre =
+    ; [TODO]: Needs to be fixed so that this is not duplicated for each test run
+    otx install -v
+allowlist_externals =
+    /bin/bash
 commands =
     coverage erase
-    - coverage run tests/fuzzing/cli_fuzzing.py {posargs:-artifact_prefix={toxworkdir}/ -print_final_stats=1 -atheris_runs=500000}
+    coverage run tests/fuzzing/cli_fuzzing.py {posargs:-artifact_prefix={toxworkdir}/ -jobs=8 -print_final_stats=1 -runs=62500 -dict={toxinidir}/tests/fuzzing/assets/cli/commands.dict}
     coverage report --precision=2
+    /bin/bash -c 'rm {toxinidir}/fuzz-*.log'