Kp/fix e2e tests (#3849)

* debug test * fixed semisl case and anomaly * fix pre-commit * updated paths * fix posix path issue * change path for semantic seg * update path * Update e2e tests for OTX2.2 validation * Fix pre-commit issue * Fix tox issue for anomaly e2e test execution. * Revert data path * Fixed an issue where the OV inference test does not work in anomaly, vp, zvp and keypoint detection tasks * Update test_cli.py --------- Co-authored-by: Emily <[email protected]>
openvinotoolkit · Aug 19, 2024 · 94d218d · 94d218d
1 parent 133fe59
commit 94d218d
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 33 deletions.
diff --git a/.github/workflows/daily.yaml b/.github/workflows/daily.yaml
@@ -22,6 +22,8 @@ jobs:
           - task: "action"
           - task: "classification"
           - task: "detection"
+          - task: "rotated_detection"
+          - task: "keypoint_detection"
           - task: "instance_segmentation"
           - task: "semantic_segmentation"
           - task: "visual_prompting"

diff --git a/tests/e2e/cli/test_cli.py b/tests/e2e/cli/test_cli.py
@@ -44,15 +44,20 @@ def test_otx_e2e_cli(
     Returns:
         None
     """
-    task = recipe.split("/")[-2].upper()
-    model_name = recipe.split("/")[-1].split(".")[0]
+    recipe_split = recipe.split("/")
+    model_name = recipe_split[-1].split(".")[0]
+    is_semisl = model_name.endswith("_semisl")
+    task = recipe_split[-2].upper() if not is_semisl else recipe_split[-3].upper()
 
     if task == OTXTaskType.INSTANCE_SEGMENTATION:
         is_tiling = "tile" in recipe
         dataset_path = fxt_target_dataset_per_task[task]["tiling" if is_tiling else "non_tiling"]
     else:
         dataset_path = fxt_target_dataset_per_task[task]
 
+    if isinstance(dataset_path, dict) and "supervised" in dataset_path:
+        dataset_path = dataset_path["supervised"]
+
     # 1) otx train
     tmp_path_train = tmp_path / f"otx_train_{model_name}"
     command_cfg = [
@@ -71,6 +76,13 @@ def test_otx_e2e_cli(
         *fxt_cli_override_command_per_task[task],
     ]
 
+    if is_semisl:
+        command_cfg.extend(
+            [
+                "--data.unlabeled_subset.data_root",
+                str(fxt_target_dataset_per_task[task]["unlabeled"]),
+            ],
+        )
     run_main(command_cfg=command_cfg, open_subprocess=fxt_open_subprocess)
 
     outputs_dir = tmp_path_train / "outputs"
@@ -121,23 +133,13 @@ def test_otx_e2e_cli(
     assert (latest_dir / "csv").exists()
 
     # 3) otx export
-    if any(
-        task_name in recipe
-        for task_name in [
-            "dino_v2",
-        ]
-    ):
-        return
+    fxt_export_list = []
     if task in ("visual_prompting", "zero_shot_visual_prompting"):
-        fxt_export_list = [
-            ExportCase2Test("ONNX", False, "exported_model_decoder.onnx"),
-            ExportCase2Test("OPENVINO", False, "exported_model_decoder.xml"),
-        ]  # TODO (sungchul): EXPORTABLE_CODE will be supported
-    elif "anomaly" in task:
-        fxt_export_list = [
-            ExportCase2Test("ONNX", False, "exported_model.onnx"),
-            ExportCase2Test("OPENVINO", False, "exported_model.xml"),
-        ]  # anomaly doesn't support exportable code
+        fxt_export_list.append(ExportCase2Test("ONNX", False, "exported_model_decoder.onnx"))
+        fxt_export_list.append(ExportCase2Test("OPENVINO", False, "exported_model_decoder.xml"))
+    elif "anomaly" in task or "keypoint_detection" in task:
+        fxt_export_list.append(ExportCase2Test("ONNX", False, "exported_model.onnx"))
+        fxt_export_list.append(ExportCase2Test("OPENVINO", False, "exported_model.xml"))
 
     overrides = fxt_cli_override_command_per_task[task]
     if "anomaly" in task:
@@ -218,11 +220,9 @@ def test_otx_e2e_cli(
     if ("_cls" not in task) and (task not in ["detection", "instance_segmentation"]):
         return  # Supported only for classification, detection and instance segmentation task.
 
-    if "dino" in model_name:
-        return  # DINO is not supported.
-
-    if "rtdetr" in model_name:
-        return  # RT-DETR currently is not supported.
+    unsupported_models = ["dino", "rtdetr"]
+    if any(model in model_name for model in unsupported_models):
+        return  # The models are not supported.
 
     tmp_path_test = tmp_path / f"otx_export_xai_{model_name}"
     for export_case in fxt_export_list:
@@ -282,8 +282,13 @@ def test_otx_explain_e2e_cli(
     """
     import cv2
 
-    task = recipe.split("/")[-2].upper()
-    model_name = recipe.split("/")[-1].split(".")[0]
+    recipe_split = recipe.split("/")
+    model_name = recipe_split[-1].split(".")[0]
+    is_semisl = model_name.endswith("_semisl")
+    task = recipe_split[-2].upper() if not is_semisl else recipe_split[-3].upper()
+
+    if is_semisl:
+        pytest.skip("SEMI-SL is not supported for explain.")
 
     if task not in [
         OTXTaskType.MULTI_CLASS_CLS,
@@ -303,6 +308,9 @@ def test_otx_explain_e2e_cli(
     else:
         dataset_path = fxt_target_dataset_per_task[task]
 
+    if isinstance(dataset_path, dict) and "supervised" in dataset_path:
+        dataset_path = dataset_path["supervised"]
+
     if "dino" in model_name:
         pytest.skip("DINO is not supported.")
 
@@ -422,6 +430,9 @@ def test_otx_hpo_e2e_cli(
     else:
         dataset_path = fxt_target_dataset_per_task[task]
 
+    if isinstance(dataset_path, dict) and "supervised" in dataset_path:
+        dataset_path = dataset_path["supervised"]
+
     tmp_path_hpo = tmp_path / f"otx_hpo_{task.lower()}"
     tmp_path_hpo.mkdir(parents=True)
 

diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
@@ -87,26 +87,33 @@ def pytest_configure(config):
 @pytest.fixture()
 def fxt_target_dataset_per_task(fxt_ci_data_root) -> dict:
     return {
-        OTXTaskType.MULTI_CLASS_CLS: Path(fxt_ci_data_root / "v2/multiclass_classification/multiclass_CUB_small/1"),
+        OTXTaskType.MULTI_CLASS_CLS: {
+            "supervised": Path(fxt_ci_data_root / "v2/multiclass_classification/multiclass_CUB_small/1"),
+            "unlabeled": Path(fxt_ci_data_root / "v2/multiclass_classification/semi-sl/CUB_unlabeled"),
+        },
         OTXTaskType.MULTI_LABEL_CLS: Path(fxt_ci_data_root / "v2/multilabel_classification/multilabel_CUB_small/1"),
         OTXTaskType.H_LABEL_CLS: Path(fxt_ci_data_root / "v2/hlabel_classification/hlabel_CUB_small/1"),
-        OTXTaskType.DETECTION: Path(fxt_ci_data_root / "v2/detection/pothole_small/1"),
-        OTXTaskType.ROTATED_DETECTION: Path(fxt_ci_data_root / "v2/detection/pothole_small/1"),
+        OTXTaskType.DETECTION: Path(fxt_ci_data_root / "v2/detection/bdd_small/1"),
+        OTXTaskType.ROTATED_DETECTION: Path(fxt_ci_data_root / "v2/rotated_detection/subway"),
         OTXTaskType.INSTANCE_SEGMENTATION: {
             "non_tiling": Path(fxt_ci_data_root / "v2/instance_seg/wgisd_small/1"),
             "tiling": Path(fxt_ci_data_root / "v2/tiling_instance_seg/vitens_aeromonas_small/1"),
         },
-        OTXTaskType.SEMANTIC_SEGMENTATION: Path(fxt_ci_data_root / "v2/semantic_seg/kvasir_small/1"),
+        OTXTaskType.SEMANTIC_SEGMENTATION: {
+            "supervised": Path(fxt_ci_data_root / "v2/semantic_seg/kvasir_small/1"),
+            "unlabeled": Path(fxt_ci_data_root / "v2/semantic_seg/semi-sl/unlabeled_images/kvasir"),
+        },
         OTXTaskType.ACTION_CLASSIFICATION: Path(
             fxt_ci_data_root / "v2/action/action_classification/ucf_kinetics_5percent_small",
         ),
-        OTXTaskType.VISUAL_PROMPTING: Path(fxt_ci_data_root / "v2/visual_prompting/wgisd_small/1"),
+        OTXTaskType.VISUAL_PROMPTING: Path(fxt_ci_data_root / "v2/visual_prompting/coco_car_person_medium"),
         OTXTaskType.ZERO_SHOT_VISUAL_PROMPTING: Path(
             fxt_ci_data_root / "v2/zero_shot_visual_prompting/coco_car_person_medium",
         ),
-        OTXTaskType.ANOMALY_CLASSIFICATION: Path(fxt_ci_data_root / "v2/anomaly/mvtec/bottle_small/1"),
+        OTXTaskType.ANOMALY_CLASSIFICATION: Path(fxt_ci_data_root / "v2/anomaly/mvtec/hazelnut_large"),
         OTXTaskType.ANOMALY_DETECTION: Path(fxt_ci_data_root / "v2/anomaly/mvtec/hazelnut_large"),
         OTXTaskType.ANOMALY_SEGMENTATION: Path(fxt_ci_data_root / "v2/anomaly/mvtec/hazelnut_large"),
+        OTXTaskType.KEYPOINT_DETECTION: Path(fxt_ci_data_root / "v2/keypoint_detection/coco_keypoint_medium"),
     }
 
 
@@ -126,4 +133,5 @@ def fxt_cli_override_command_per_task() -> dict:
         OTXTaskType.ANOMALY_CLASSIFICATION: [],
         OTXTaskType.ANOMALY_DETECTION: [],
         OTXTaskType.ANOMALY_SEGMENTATION: [],
+        OTXTaskType.KEYPOINT_DETECTION: [],
     }
diff --git a/tox.ini b/tox.ini
@@ -17,11 +17,13 @@ task =
     hlabel_classification: "hlabel_classification"
     detection: "detection"
     rotated_detection: "rotated_detection"
+    keypoint_detection: "keypoint_detection"
     instance_segmentation: "instance_segmentation"
     semantic_segmentation: "semantic_segmentation"
     visual_prompting_all: "visual_prompting_all"
     visual_prompting: "visual_prompting"
     zero_shot_visual_prompting: "visual_prompting"
+    anomaly: "anomaly"
     anomaly_classification: "anomaly_classification"
     anomaly_detection: "anomaly_detection"
     anomaly_segmentation: "anomaly_segmentation"
@@ -53,7 +55,7 @@ commands =
         {posargs}
 
 
-[testenv:integration-test-{all, action, classification, multi_cls_classification, multi_label_classification, hlabel_classification, detection, rotated_detection, instance_segmentation, semantic_segmentation, visual_prompting_all, visual_prompting, zero_shot_visual_prompting, anomaly_classification, anomaly_detection, anomaly_segmentation}]
+[testenv:integration-test-{all, action, classification, multi_cls_classification, multi_label_classification, hlabel_classification, detection, rotated_detection, keypoint_detection, instance_segmentation, semantic_segmentation, visual_prompting_all, visual_prompting, zero_shot_visual_prompting, anomaly_classification, anomaly_detection, anomaly_segmentation}]
 setenv =
     CUBLAS_WORKSPACE_CONFIG=:4096:8
 deps =
@@ -62,7 +64,7 @@ commands =
     python -m pytest tests/integration -ra --showlocals --csv={toxworkdir}/{envname}.csv --task {[testenv]task} --open-subprocess {posargs}
 
 
-[testenv:e2e-test-{all, action, classification, detection, rotated_detection, instance_segmentation, semantic_segmentation, visual_prompting, anomaly}]
+[testenv:e2e-test-{all, action, classification, detection, rotated_detection, keypoint_detection, instance_segmentation, semantic_segmentation, visual_prompting, anomaly}]
 setenv =
     CUBLAS_WORKSPACE_CONFIG=:4096:8
 deps =