Merge pull request #1232 from openvinotoolkit/eugene/CVS-90192-fix-sa…

…liency-map

Fix saliency map issues

external/deep-object-reid/torchreid_tasks/inference_task.py

@@ -48,15 +48,16 @@
     check_input_parameters_type,
 )
 from ote_sdk.utils.labels_utils import get_empty_label
+from ote_sdk.utils.vis_utils import get_actmap
 from scripts.default_config import (get_default_config, imagedata_kwargs,
                                     merge_from_files_with_base, model_kwargs)
 from torchreid.apis.export import export_ir, export_onnx
 from torchreid_tasks.monitors import DefaultMetricsMonitor, StopCallback
 from torchreid_tasks.parameters import OTEClassificationParameters
-from torchreid_tasks.utils import (active_score_from_probs, force_fp32, get_actmap, get_multiclass_predictions,
-                                            get_multilabel_predictions, InferenceProgressCallback,
-                                            OTEClassificationDataset, sigmoid_numpy, softmax_numpy,
-                                            get_multihead_class_info, get_hierarchical_predictions)
+from torchreid_tasks.utils import (active_score_from_probs, force_fp32, get_multiclass_predictions,
+                                   get_multilabel_predictions, InferenceProgressCallback,
+                                   OTEClassificationDataset, sigmoid_numpy, softmax_numpy,
+                                   get_multihead_class_info, get_hierarchical_predictions)
 from torchreid.metrics.classification import score_extraction
 from torchreid.utils import load_pretrained_weights
 
@@ -262,9 +263,9 @@ def infer(self, dataset: DatasetEntity,
 
             if dump_features:
                 actmap = get_actmap(saliency_maps[i], (dataset_item.width, dataset_item.height))
-                saliency_media = ResultMediaEntity(name="saliency_map", type="Saliency map",
+                saliency_media = ResultMediaEntity(name="Saliency Map", type="saliency_map",
                                                    annotation_scene=dataset_item.annotation_scene,
-                                                   numpy=actmap, roi=dataset_item.roi, label = item_labels[0].label)
+                                                   numpy=actmap, roi=dataset_item.roi, label=item_labels[0].label)
                 dataset_item.append_metadata_item(saliency_media, model=self._task_environment.model)
 
         return dataset

external/deep-object-reid/torchreid_tasks/openvino_task.py

@@ -191,7 +191,7 @@ def infer(self, dataset: DatasetEntity,
             feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1))
             dataset_item.append_metadata_item(feature_vec_media, model=self.model)
             if dump_features:
-                saliency_media = ResultMediaEntity(name="saliency_map", type="Saliency map",
+                saliency_media = ResultMediaEntity(name="Saliency Map", type="saliency_map",
                                                    annotation_scene=dataset_item.annotation_scene,
                                                    numpy=actmap, roi=dataset_item.roi,
                                                    label=predicted_scene.annotations[0].get_labels()[0].label)

external/deep-object-reid/torchreid_tasks/utils.py

@@ -497,15 +497,6 @@ def on_initialization_end(self):
         self.update_progress_callback(self.get_progress())
 
 
-@check_input_parameters_type()
-def get_actmap(features: Union[np.ndarray, Iterable, int, float],
-               output_res: Union[tuple, list]):
-    am = cv.resize(features, output_res)
-    am = cv.applyColorMap(am, cv.COLORMAP_JET)
-    am = cv.cvtColor(am, cv.COLOR_BGR2RGB)
-    return am
-
-
 @check_input_parameters_type()
 def active_score_from_probs(predictions: Union[np.ndarray, Iterable, int, float]):
     top_idxs = np.argpartition(predictions, -2)[-2:]

external/mmdetection/detection_tasks/apis/detection/inference_task.py

@@ -51,6 +51,7 @@
     DatasetParamTypeCheck,
     check_input_parameters_type,
 )
+from ote_sdk.utils.vis_utils import get_actmap
 
 from mmdet.apis import export_model
 from detection_tasks.apis.detection.config_utils import patch_config, prepare_for_testing, set_hyperparams
@@ -250,13 +251,12 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, confidence_th
             if feature_vector is not None:
                 active_score = TensorEntity(name="representation_vector", numpy=feature_vector)
                 dataset_item.append_metadata_item(active_score, model=self._task_environment.model)
-            
+
             if saliency_map is not None:
-                width, height = dataset_item.width, dataset_item.height
-                saliency_map = cv2.resize(saliency_map, (width, height), interpolation=cv2.INTER_NEAREST)
-                saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map",
-                                                annotation_scene=dataset_item.annotation_scene, 
-                                                numpy=saliency_map, roi=dataset_item.roi)
+                saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height))
+                saliency_map_media = ResultMediaEntity(name="Saliency Map", type="saliency_map",
+                                                       annotation_scene=dataset_item.annotation_scene,
+                                                       numpy=saliency_map, roi=dataset_item.roi)
                 dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model)
 
 
@@ -292,7 +292,7 @@ def hook(module, input, output):
         model = self._model
         with model.register_forward_pre_hook(pre_hook), model.register_forward_hook(hook):
             prediction_results, _ = self._infer_detector(model, self._config, dataset, dump_features=True, eval=False, 
-                                                        dump_saliency_map=dump_saliency_map)
+                                                         dump_saliency_map=dump_saliency_map)
         self._add_predictions_to_dataset(prediction_results, dataset, self.confidence_threshold)
 
         logger.info('Inference completed')
@@ -337,8 +337,8 @@ def dump_saliency_hook(model: torch.nn.Module, input: Tuple, out: List[torch.Ten
 
             Args:
                 model (torch.nn.Module): PyTorch model
-                input (Tuple): input 
-                out (List[torch.Tensor]): a list of feature maps 
+                input (Tuple): input
+                out (List[torch.Tensor]): a list of feature maps
             """
             with torch.no_grad():
                 saliency_map = get_saliency_map(out[-1])

external/mmdetection/detection_tasks/apis/detection/openvino_task.py

@@ -14,7 +14,6 @@
 
 import attr
 import copy
-import cv2
 import io
 import json
 import numpy as np
@@ -64,6 +63,7 @@
     DatasetParamTypeCheck,
     check_input_parameters_type,
 )
+from ote_sdk.utils.vis_utils import get_actmap
 from shutil import copyfile, copytree
 from typing import Any, Dict, List, Optional, Tuple, Union
 from zipfile import ZipFile
@@ -286,11 +286,10 @@ def infer(self, dataset: DatasetEntity, inference_parameters: Optional[Inference
                 dataset_item.append_metadata_item(representation_vector, model=self.model)
 
             if add_saliency_map and saliency_map is not None:
-                width, height = dataset_item.width, dataset_item.height
-                saliency_map = cv2.resize(saliency_map[0], (width, height), interpolation=cv2.INTER_NEAREST)
-                saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map",
-                                                annotation_scene=dataset_item.annotation_scene, 
-                                                numpy=saliency_map, roi=dataset_item.roi)
+                saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height))
+                saliency_map_media = ResultMediaEntity(name="Saliency Map", type="saliency_map",
+                                                       annotation_scene=dataset_item.annotation_scene,
+                                                       numpy=saliency_map, roi=dataset_item.roi)
                 dataset_item.append_metadata_item(saliency_map_media, model=self.model)
         logger.info('OpenVINO inference completed')
         return dataset

external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py

@@ -49,7 +49,7 @@
 )
 
 from mmseg.apis import export_model
-from mmseg.core.hooks.auxiliary_hooks import FeatureVectorHook, SaliencyMapHook
+from mmseg.core.hooks.auxiliary_hooks import FeatureVectorHook
 from mmseg.datasets import build_dataloader, build_dataset
 from mmseg.models import build_segmentor
 from mmseg.parallel import MMDataCPU
@@ -196,16 +196,15 @@ def hook(module, input, output):
         pre_hook_handle = self._model.register_forward_pre_hook(pre_hook)
         hook_handle = self._model.register_forward_hook(hook)
 
-        prediction_results = self._infer_segmentor(self._model, self._config, dataset, dump_features=True,
-                                                   dump_saliency_map=not is_evaluation)
-        self._add_predictions_to_dataset(prediction_results, dataset)
+        prediction_results = self._infer_segmentor(self._model, self._config, dataset, dump_features=True)
+        self._add_predictions_to_dataset(prediction_results, dataset, dump_soft_prediction=not is_evaluation)
         pre_hook_handle.remove()
         hook_handle.remove()
 
         return dataset
 
-    def _add_predictions_to_dataset(self, prediction_results, dataset):
-        for dataset_item, (prediction, feature_vector, saliency_map) in zip(dataset, prediction_results):
+    def _add_predictions_to_dataset(self, prediction_results, dataset, dump_soft_prediction):
+        for dataset_item, (prediction, feature_vector) in zip(dataset, prediction_results):
             soft_prediction = np.transpose(prediction, axes=(1, 2, 0))
             hard_prediction = create_hard_prediction_from_soft_prediction(
                 soft_prediction=soft_prediction,
@@ -223,18 +222,23 @@ def _add_predictions_to_dataset(self, prediction_results, dataset):
                 active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1))
                 dataset_item.append_metadata_item(active_score, model=self._task_environment.model)
 
-            if saliency_map is not None:
-                class_act_map = get_activation_map(saliency_map, (dataset_item.width, dataset_item.height))
-                result_media = ResultMediaEntity(name="saliency_map",
-                                                 type="Saliency map",
-                                                 annotation_scene=dataset_item.annotation_scene,
-                                                 roi=dataset_item.roi,
-                                                 numpy=class_act_map)
-                dataset_item.append_metadata_item(result_media, model=self._task_environment.model)
-
-    def _infer_segmentor(self,
-                         model: torch.nn.Module, config: Config, dataset: DatasetEntity,
-                         dump_features: bool = False, dump_saliency_map: bool = False) -> None:
+            if dump_soft_prediction:
+                for label_index, label in self._label_dictionary.items():
+                    if label_index == 0:
+                        continue
+                    current_label_soft_prediction = soft_prediction[:, :, label_index]
+
+                    class_act_map = get_activation_map(current_label_soft_prediction)
+                    result_media = ResultMediaEntity(name='Soft Prediction',
+                                                     type='soft_prediction',
+                                                     label=label,
+                                                     annotation_scene=dataset_item.annotation_scene,
+                                                     roi=dataset_item.roi,
+                                                     numpy=class_act_map)
+                    dataset_item.append_metadata_item(result_media, model=self._task_environment.model)
+
+    def _infer_segmentor(self, model: torch.nn.Module, config: Config, dataset: DatasetEntity,
+                         dump_features: bool = False) -> None:
         model.eval()
 
         test_config = prepare_for_testing(config, dataset)
@@ -254,21 +258,18 @@ def _infer_segmentor(self,
 
         eval_predictions = []
         feature_vectors = []
-        saliency_maps = []
 
         # Use a single gpu for testing. Set in both mm_val_dataloader and eval_model
         with FeatureVectorHook(model.module.backbone) if dump_features else nullcontext() as fhook:
-            with SaliencyMapHook(model.module.backbone) if dump_saliency_map else nullcontext() as shook:
-                for data in mm_val_dataloader:
-                    with torch.no_grad():
-                        result = model(return_loss=False, output_logits=True, **data)
-                    eval_predictions.extend(result)
-                feature_vectors = fhook.records if dump_features else [None] * len(dataset)
-                saliency_maps = shook.records if dump_saliency_map else [None] * len(dataset)
-        assert len(eval_predictions) == len(feature_vectors) == len(saliency_maps), \
+            for data in mm_val_dataloader:
+                with torch.no_grad():
+                    result = model(return_loss=False, output_logits=True, **data)
+                eval_predictions.extend(result)
+            feature_vectors = fhook.records if dump_features else [None] * len(dataset)
+        assert len(eval_predictions) == len(feature_vectors), \
                'Number of elements should be the same, however, number of outputs are ' \
-               f"{len(eval_predictions)}, {len(feature_vectors)}, and {len(saliency_maps)}"
-        predictions = zip(eval_predictions, feature_vectors, saliency_maps)
+               f"{len(eval_predictions)} and {len(feature_vectors)}"
+        predictions = zip(eval_predictions, feature_vectors)
         return predictions
 
     @check_input_parameters_type()

external/mmsegmentation/segmentation_tasks/apis/segmentation/model_wrappers/blur.py

@@ -26,13 +26,16 @@
 
 
 @check_input_parameters_type()
-def get_actmap(
-    features: Union[np.ndarray, Iterable, int, float], output_res: Union[tuple, list]
-):
-    am = cv2.resize(features, output_res)
-    am = cv2.applyColorMap(am, cv2.COLORMAP_JET)
-    am = cv2.cvtColor(am, cv2.COLOR_BGR2RGB)
-    return am
+def get_activation_map(features: Union[np.ndarray, Iterable, int, float]):
+    min_soft_score = np.min(features)
+    max_soft_score = np.max(features)
+    factor = 255.0 / (max_soft_score - min_soft_score + 1e-12)
+
+    float_act_map = factor * (features - min_soft_score)
+    int_act_map = np.uint8(np.floor(float_act_map))
+    int_act_map = cv2.applyColorMap(int_act_map, cv2.COLORMAP_JET)
+    int_act_map = cv2.cvtColor(int_act_map, cv2.COLOR_BGR2RGB)
+    return int_act_map
 
 
 class BlurSegmentation(SegmentationModel):
@@ -78,18 +81,16 @@ def postprocess(self, outputs: Dict[str, np.ndarray], metadata: Dict[str, Any]):
             soft_threshold=self.soft_threshold,
             blur_strength=self.blur_strength
         )
-        hard_prediction = cv2.resize(hard_prediction, metadata['original_shape'][1::-1], 0, 0, interpolation=cv2.INTER_NEAREST)
-
-        if 'feature_vector' not in outputs or 'saliency_map' not in outputs:
-            warnings.warn('Could not find Feature Vector and Saliency Map in OpenVINO output. '
-                'Please rerun OpenVINO export or retrain the model.')
-            metadata["saliency_map"] = None
+        hard_prediction = cv2.resize(hard_prediction, metadata['original_shape'][1::-1], 0, 0,
+                                     interpolation=cv2.INTER_NEAREST)
+        soft_prediction = cv2.resize(soft_prediction, metadata['original_shape'][1::-1], 0, 0,
+                                     interpolation=cv2.INTER_NEAREST)
+        metadata['soft_prediction'] = soft_prediction
+
+        if 'feature_vector' not in outputs:
+            warnings.warn('Could not find Feature Vector in OpenVINO output. Please rerun export or retrain the model.')
             metadata["feature_vector"] = None
         else:
-            metadata["saliency_map"] = get_actmap(
-                outputs["saliency_map"][0],
-                (metadata["original_shape"][1], metadata["original_shape"][0]),
-            )
             metadata["feature_vector"] = outputs["feature_vector"].reshape(-1)
 
         return hard_prediction

external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py

@@ -105,11 +105,11 @@ def pre_process(self, image: np.ndarray) -> Tuple[Dict[str, np.ndarray], Dict[st
     @check_input_parameters_type()
     def post_process(self, prediction: Dict[str, np.ndarray], metadata: Dict[str, Any]) -> AnnotationSceneEntity:
         hard_prediction = self.model.postprocess(prediction, metadata)
+        soft_prediction = metadata['soft_prediction']
         feature_vector = metadata['feature_vector']
-        saliency_map = metadata['saliency_map']
         predicted_scene = self.converter.convert_to_annotation(hard_prediction, metadata)
 
-        return predicted_scene, feature_vector, saliency_map
+        return predicted_scene, feature_vector, soft_prediction
 
     @check_input_parameters_type()
     def forward(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]:
@@ -164,25 +164,33 @@ def infer(self,
               inference_parameters: Optional[InferenceParameters] = None) -> DatasetEntity:
         if inference_parameters is not None:
             update_progress_callback = inference_parameters.update_progress
-            dump_saliency_map = not inference_parameters.is_evaluation
+            dump_soft_prediction = not inference_parameters.is_evaluation
         else:
             update_progress_callback = default_progress_callback
-            dump_saliency_map = True
+            dump_soft_prediction = True
 
         dataset_size = len(dataset)
         for i, dataset_item in enumerate(dataset, 1):
-            predicted_scene, feature_vector, saliency_map = self.inferencer.predict(dataset_item.numpy)
+            predicted_scene, feature_vector, soft_prediction = self.inferencer.predict(dataset_item.numpy)
             dataset_item.append_annotations(predicted_scene.annotations)
 
             if feature_vector is not None:
                 feature_vector_media = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1))
                 dataset_item.append_metadata_item(feature_vector_media, model=self.model)
 
-            if dump_saliency_map and saliency_map is not None:
-                saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map",
-                                                annotation_scene=dataset_item.annotation_scene, 
-                                                numpy=saliency_map, roi=dataset_item.roi)
-                dataset_item.append_metadata_item(saliency_map_media, model=self.model)
+            if dump_soft_prediction:
+                for label_index, label in self._label_dictionary.items():
+                    if label_index == 0:
+                        continue
+                    current_label_soft_prediction = soft_prediction[:, :, label_index]
+                    class_act_map = get_activation_map(current_label_soft_prediction)
+                    result_media = ResultMediaEntity(name='Soft Prediction',
+                                                     type='soft_prediction',
+                                                     label=label,
+                                                     annotation_scene=dataset_item.annotation_scene,
+                                                     roi=dataset_item.roi,
+                                                     numpy=class_act_map)
+                    dataset_item.append_metadata_item(result_media, model=self.model)
 
             update_progress_callback(int(i / dataset_size * 100))