Add e2e test for visual prompting (#2360)

* (WIP) otx optimize * pre-commit * (WIP) set e2e * Remove nncf config * Add visual prompting requirement * Add visual prompting in tox * Add visual prompting in setup.py * Fix typo * Delete unused configuration.yaml * Edit test_name * Add to limit activation range * Update from `vp` to `visprompt` * Fix about no returning the first label * pre-commit * (WIP) otx optimize * pre-commit * (WIP) set e2e * Remove nncf config * Add visual prompting requirement * Add visual prompting in tox * Add visual prompting in setup.py * Fix typo * pre-commit * Add actions * Update tests/e2e/cli/visual_prompting/test_visual_prompting.py Co-authored-by: Jaeguk Hyun <[email protected]> * Skip PTQ e2e test * Change task name * Remove skipped tc --------- Co-authored-by: Jaeguk Hyun <[email protected]>
openvinotoolkit · Jul 18, 2023 · edd1a8c · edd1a8c
1 parent 3d157ab
commit edd1a8c
Show file tree

Hide file tree

Showing 15 changed files with 229 additions and 247 deletions.
diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml
@@ -24,6 +24,8 @@ jobs:
             test_dir: "tests/e2e/cli/instance_segmentation"
           - task: "seg"
             test_dir: "tests/e2e/cli/semantic_segmentation"
+          - task: "visprompt"
+            test_dir: "tests/e2e/cli/visual_prompting"
     name: E2E-Test-py310-${{ matrix.task }}
     uses: ./.github/workflows/run_tests_in_tox.yml
     with:

diff --git a/.github/workflows/pre_merge.yml b/.github/workflows/pre_merge.yml
@@ -104,6 +104,8 @@ jobs:
             test_dir: "tests/integration/cli/action"
           - task: "ano"
             test_dir: "tests/integration/cli/anomaly"
+          - task: "visprompt"
+            test_dir: "tests/integration/cli/visual_prompting"
     name: Integration-Test-py310-${{ matrix.task }}
     # This is what will cancel the job concurrency
     concurrency:

diff --git a/requirements/visual_prompting.txt b/requirements/visual_prompting.txt
@@ -0,0 +1,4 @@
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# Visual Prompting Requirements.
+scikit-image
+pytorch-lightning>=1.7.0,<1.10.0
diff --git a/setup.py b/setup.py
@@ -149,13 +149,18 @@ def _cython_modules():
             "base", "openvino", "segmentation",
         ]
     ),
+    "visual_prompting": get_requirements(requirement_files=[
+            "base", "openvino", "visual_prompting",
+        ]
+    ),
     "full": get_requirements(requirement_files=[
             "base",
             "openvino",
             "anomaly",
             "classification",
             "detection",
             "segmentation",
+            "visual_prompting",
             "action",
         ]
     ),

diff --git a/src/otx/algorithms/visual_prompting/configs/configuration.yaml b/src/otx/algorithms/visual_prompting/configs/configuration.yaml
@@ -85,62 +85,6 @@ learning_parameters:
       visible_in_ui: true
       warning: null
       auto_hpo_state: NOT_POSSIBLE
-nncf_optimization:
-  description: Optimization by NNCF
-  enable_pruning:
-    affects_outcome_of: NONE
-    auto_hpo_state: not_possible
-    auto_hpo_value: null
-    default_value: false
-    description: Enable filter pruning algorithm
-    editable: true
-    header: Enable filter pruning algorithm
-    type: BOOLEAN
-    ui_rules:
-      action: DISABLE_EDITING
-      operator: AND
-      rules: []
-      type: UI_RULES
-    value: false
-    visible_in_ui: true
-    warning: null
-  enable_quantization:
-    affects_outcome_of: NONE
-    auto_hpo_state: not_possible
-    auto_hpo_value: null
-    default_value: true
-    description: Enable quantization algorithm
-    editable: true
-    header: Enable quantization algorithm
-    type: BOOLEAN
-    ui_rules:
-      action: DISABLE_EDITING
-      operator: AND
-      rules: []
-      type: UI_RULES
-    value: true
-    visible_in_ui: true
-    warning: null
-  header: Optimization by NNCF
-  pruning_supported:
-    affects_outcome_of: TRAINING
-    auto_hpo_state: not_possible
-    auto_hpo_value: null
-    default_value: false
-    description: Whether filter pruning is supported
-    editable: false
-    header: Whether filter pruning is supported
-    type: BOOLEAN
-    ui_rules:
-      action: DISABLE_EDITING
-      operator: AND
-      rules: []
-      type: UI_RULES
-    value: false
-    visible_in_ui: false
-    warning: null
-  type: PARAMETER_GROUP
-  visible_in_ui: true
 pot_parameters:
   description: POT Parameters
   header: POT Parameters

diff --git a/src/otx/algorithms/visual_prompting/configs/sam_vit_b/configuration.yaml b/src/otx/algorithms/visual_prompting/configs/sam_vit_b/configuration.yaml
diff --git a/src/otx/algorithms/visual_prompting/configs/sam_vit_b/ptq_optimization_config.py b/src/otx/algorithms/visual_prompting/configs/sam_vit_b/ptq_optimization_config.py
@@ -0,0 +1,23 @@
+"""PTQ config file."""
+from nncf.common.quantization.structs import QuantizationPreset
+from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters
+from nncf.quantization.range_estimator import (
+    AggregatorType,
+    RangeEstimatorParameters,
+    StatisticsCollectorParameters,
+    StatisticsType,
+)
+
+advanced_parameters = AdvancedQuantizationParameters(
+    activations_range_estimator_params=RangeEstimatorParameters(
+        min=StatisticsCollectorParameters(
+            statistics_type=StatisticsType.QUANTILE, aggregator_type=AggregatorType.MIN, quantile_outlier_prob=1e-4
+        ),
+        max=StatisticsCollectorParameters(
+            statistics_type=StatisticsType.QUANTILE, aggregator_type=AggregatorType.MAX, quantile_outlier_prob=1e-4
+        ),
+    ),
+    # backend_params={"use_pot": True},
+)
+
+preset = QuantizationPreset.MIXED
diff --git a/src/otx/algorithms/visual_prompting/tasks/openvino.py b/src/otx/algorithms/visual_prompting/tasks/openvino.py
@@ -28,13 +28,14 @@
 import nncf
 import numpy as np
 import openvino.runtime as ov
+from addict import Dict as ADDict
 from nncf.common.quantization.structs import QuantizationPreset
 from openvino.model_api.adapters import OpenvinoAdapter, create_core
 from openvino.model_api.models import Model
 
+from otx.algorithms.common.utils import get_default_async_reqs_num, read_py_config
 from otx.algorithms.common.utils.ir import check_if_quantized
 from otx.algorithms.common.utils.logger import get_logger
-from otx.algorithms.common.utils.utils import get_default_async_reqs_num
 from otx.algorithms.visual_prompting.adapters.openvino import model_wrappers
 from otx.algorithms.visual_prompting.adapters.pytorch_lightning.datasets.dataset import (
     OTXVisualPromptingDataset,
@@ -447,13 +448,17 @@ def optimize(
             if optimization_parameters is not None:
                 optimization_parameters.update_progress(10 * i + 35 * (i - 1), None)
 
-            stat_subset_size = self.hparams.pot_parameters.stat_subset_size
-            preset = QuantizationPreset(self.hparams.pot_parameters.preset.name.lower())
-
-            compressed_model = nncf.quantize(
-                ov_model, quantization_dataset, subset_size=min(stat_subset_size, len(data_loader)), preset=preset
+            optimization_config_path = os.path.join(self._base_dir, "ptq_optimization_config.py")
+            ptq_config = ADDict()
+            if os.path.exists(optimization_config_path):
+                ptq_config = read_py_config(optimization_config_path)
+            ptq_config.update(
+                subset_size=min(self.hparams.pot_parameters.stat_subset_size, len(data_loader)),
+                preset=QuantizationPreset(self.hparams.pot_parameters.preset.name.lower()),
             )
 
+            compressed_model = nncf.quantize(ov_model, quantization_dataset, **ptq_config)
+
             if optimization_parameters is not None:
                 optimization_parameters.update_progress(45 * i, None)
 

diff --git a/src/otx/core/data/adapter/visual_prompting_dataset_adapter.py b/src/otx/core/data/adapter/visual_prompting_dataset_adapter.py
@@ -60,7 +60,7 @@ def get_otx_dataset(self) -> DatasetEntity:
                             if self.use_mask:
                                 # use masks loaded in datumaro as-is
                                 if self.data_type == "common_semantic_segmentation":
-                                    if new_label := self.updated_label_id.get(ann.label, None):
+                                    if (new_label := self.updated_label_id.get(ann.label, None)) is not None:
                                         ann.label = new_label
                                     else:
                                         continue
@@ -70,7 +70,7 @@ def get_otx_dataset(self) -> DatasetEntity:
                                 # convert masks to polygons, they will be converted to masks again
                                 datumaro_polygons = MasksToPolygons.convert_mask(ann)
                                 for d_polygon in datumaro_polygons:
-                                    if new_label := self.updated_label_id.get(d_polygon.label, None):
+                                    if (new_label := self.updated_label_id.get(d_polygon.label, None)) is not None:
                                         d_polygon.label = new_label
                                     else:
                                         continue

diff --git a/tests/e2e/cli/visual_prompting/__init__.py b/tests/e2e/cli/visual_prompting/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
diff --git a/tests/e2e/cli/visual_prompting/reference/Visual_Prompting_SAM_ViT_B/compressed_model.yml b/tests/e2e/cli/visual_prompting/reference/Visual_Prompting_SAM_ViT_B/compressed_model.yml
@@ -0,0 +1,3 @@
+TestToolsVisualPrompting:
+  pot:
+    number_of_fakequantizers: 210