From 487b46737d6bcf84a5ff38be83463e82c495953c Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Fri, 26 Aug 2022 16:27:08 +0100 Subject: [PATCH 1/5] fix saliency map issues --- .../torchreid_tasks/inference_task.py | 13 ++-- .../torchreid_tasks/openvino_task.py | 2 +- .../deep-object-reid/torchreid_tasks/utils.py | 9 --- .../apis/detection/inference_task.py | 18 +++--- .../apis/detection/openvino_task.py | 11 ++-- external/mmdetection/submodule | 2 +- .../apis/segmentation/inference_task.py | 62 ++++++++++--------- .../apis/segmentation/model_wrappers/blur.py | 35 ++++++----- .../apis/segmentation/openvino_task.py | 31 +++++++--- .../apis/segmentation/ote_utils.py | 15 +++-- external/mmsegmentation/submodule | 2 +- .../mpa_tasks/apis/classification/task.py | 4 +- .../mpa_tasks/apis/detection/task.py | 7 ++- .../mpa_tasks/apis/segmentation/task.py | 40 +++++++----- .../mpa_tasks/utils/data_utils.py | 13 ---- .../model-preparation-algorithm/submodule | 2 +- ote_sdk/ote_sdk/utils/vis_utils.py | 26 ++++++++ 17 files changed, 163 insertions(+), 129 deletions(-) create mode 100644 ote_sdk/ote_sdk/utils/vis_utils.py diff --git a/external/deep-object-reid/torchreid_tasks/inference_task.py b/external/deep-object-reid/torchreid_tasks/inference_task.py index adb4c4acced..bca5795c3a3 100644 --- a/external/deep-object-reid/torchreid_tasks/inference_task.py +++ b/external/deep-object-reid/torchreid_tasks/inference_task.py @@ -48,15 +48,16 @@ check_input_parameters_type, ) from ote_sdk.utils.labels_utils import get_empty_label +from ote_sdk.utils.vis_utils import get_actmap from scripts.default_config import (get_default_config, imagedata_kwargs, merge_from_files_with_base, model_kwargs) from torchreid.apis.export import export_ir, export_onnx from torchreid_tasks.monitors import DefaultMetricsMonitor, StopCallback from torchreid_tasks.parameters import OTEClassificationParameters -from torchreid_tasks.utils import (active_score_from_probs, force_fp32, get_actmap, get_multiclass_predictions, - get_multilabel_predictions, InferenceProgressCallback, - OTEClassificationDataset, sigmoid_numpy, softmax_numpy, - get_multihead_class_info, get_hierarchical_predictions) +from torchreid_tasks.utils import (active_score_from_probs, force_fp32, get_multiclass_predictions, + get_multilabel_predictions, InferenceProgressCallback, + OTEClassificationDataset, sigmoid_numpy, softmax_numpy, + get_multihead_class_info, get_hierarchical_predictions) from torchreid.metrics.classification import score_extraction from torchreid.utils import load_pretrained_weights @@ -262,9 +263,9 @@ def infer(self, dataset: DatasetEntity, if dump_features: actmap = get_actmap(saliency_maps[i], (dataset_item.width, dataset_item.height)) - saliency_media = ResultMediaEntity(name="saliency_map", type="Saliency map", + saliency_media = ResultMediaEntity(name="Saliency Map", type="saliency_map", annotation_scene=dataset_item.annotation_scene, - numpy=actmap, roi=dataset_item.roi, label = item_labels[0].label) + numpy=actmap, roi=dataset_item.roi, label=item_labels[0].label) dataset_item.append_metadata_item(saliency_media, model=self._task_environment.model) return dataset diff --git a/external/deep-object-reid/torchreid_tasks/openvino_task.py b/external/deep-object-reid/torchreid_tasks/openvino_task.py index 43e749a7792..c872ca83230 100644 --- a/external/deep-object-reid/torchreid_tasks/openvino_task.py +++ b/external/deep-object-reid/torchreid_tasks/openvino_task.py @@ -191,7 +191,7 @@ def infer(self, dataset: DatasetEntity, feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1)) dataset_item.append_metadata_item(feature_vec_media, model=self.model) if dump_features: - saliency_media = ResultMediaEntity(name="saliency_map", type="Saliency map", + saliency_media = ResultMediaEntity(name="Saliency Map", type="saliency_map", annotation_scene=dataset_item.annotation_scene, numpy=actmap, roi=dataset_item.roi, label=predicted_scene.annotations[0].get_labels()[0].label) diff --git a/external/deep-object-reid/torchreid_tasks/utils.py b/external/deep-object-reid/torchreid_tasks/utils.py index 5b2b9b7ce3f..aed9ad4794e 100644 --- a/external/deep-object-reid/torchreid_tasks/utils.py +++ b/external/deep-object-reid/torchreid_tasks/utils.py @@ -497,15 +497,6 @@ def on_initialization_end(self): self.update_progress_callback(self.get_progress()) -@check_input_parameters_type() -def get_actmap(features: Union[np.ndarray, Iterable, int, float], - output_res: Union[tuple, list]): - am = cv.resize(features, output_res) - am = cv.applyColorMap(am, cv.COLORMAP_JET) - am = cv.cvtColor(am, cv.COLOR_BGR2RGB) - return am - - @check_input_parameters_type() def active_score_from_probs(predictions: Union[np.ndarray, Iterable, int, float]): top_idxs = np.argpartition(predictions, -2)[-2:] diff --git a/external/mmdetection/detection_tasks/apis/detection/inference_task.py b/external/mmdetection/detection_tasks/apis/detection/inference_task.py index 5670dbdf7e6..358b09a9750 100644 --- a/external/mmdetection/detection_tasks/apis/detection/inference_task.py +++ b/external/mmdetection/detection_tasks/apis/detection/inference_task.py @@ -51,6 +51,7 @@ DatasetParamTypeCheck, check_input_parameters_type, ) +from ote_sdk.utils.vis_utils import get_actmap from mmdet.apis import export_model from detection_tasks.apis.detection.config_utils import patch_config, prepare_for_testing, set_hyperparams @@ -250,13 +251,12 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, confidence_th if feature_vector is not None: active_score = TensorEntity(name="representation_vector", numpy=feature_vector) dataset_item.append_metadata_item(active_score, model=self._task_environment.model) - + if saliency_map is not None: - width, height = dataset_item.width, dataset_item.height - saliency_map = cv2.resize(saliency_map, (width, height), interpolation=cv2.INTER_NEAREST) - saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map", - annotation_scene=dataset_item.annotation_scene, - numpy=saliency_map, roi=dataset_item.roi) + saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) + saliency_map_media = ResultMediaEntity(name="Saliency Map", type="saliency_map", + annotation_scene=dataset_item.annotation_scene, + numpy=saliency_map, roi=dataset_item.roi) dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) @@ -292,7 +292,7 @@ def hook(module, input, output): model = self._model with model.register_forward_pre_hook(pre_hook), model.register_forward_hook(hook): prediction_results, _ = self._infer_detector(model, self._config, dataset, dump_features=True, eval=False, - dump_saliency_map=dump_saliency_map) + dump_saliency_map=dump_saliency_map) self._add_predictions_to_dataset(prediction_results, dataset, self.confidence_threshold) logger.info('Inference completed') @@ -337,8 +337,8 @@ def dump_saliency_hook(model: torch.nn.Module, input: Tuple, out: List[torch.Ten Args: model (torch.nn.Module): PyTorch model - input (Tuple): input - out (List[torch.Tensor]): a list of feature maps + input (Tuple): input + out (List[torch.Tensor]): a list of feature maps """ with torch.no_grad(): saliency_map = get_saliency_map(out[-1]) diff --git a/external/mmdetection/detection_tasks/apis/detection/openvino_task.py b/external/mmdetection/detection_tasks/apis/detection/openvino_task.py index 0d57da2df21..2232cf90d8a 100644 --- a/external/mmdetection/detection_tasks/apis/detection/openvino_task.py +++ b/external/mmdetection/detection_tasks/apis/detection/openvino_task.py @@ -14,7 +14,6 @@ import attr import copy -import cv2 import io import json import numpy as np @@ -64,6 +63,7 @@ DatasetParamTypeCheck, check_input_parameters_type, ) +from ote_sdk.utils.vis_utils import get_actmap from shutil import copyfile, copytree from typing import Any, Dict, List, Optional, Tuple, Union from zipfile import ZipFile @@ -286,11 +286,10 @@ def infer(self, dataset: DatasetEntity, inference_parameters: Optional[Inference dataset_item.append_metadata_item(representation_vector, model=self.model) if add_saliency_map and saliency_map is not None: - width, height = dataset_item.width, dataset_item.height - saliency_map = cv2.resize(saliency_map[0], (width, height), interpolation=cv2.INTER_NEAREST) - saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map", - annotation_scene=dataset_item.annotation_scene, - numpy=saliency_map, roi=dataset_item.roi) + saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) + saliency_map_media = ResultMediaEntity(name="Saliency Map", type="saliency_map", + annotation_scene=dataset_item.annotation_scene, + numpy=saliency_map, roi=dataset_item.roi) dataset_item.append_metadata_item(saliency_map_media, model=self.model) logger.info('OpenVINO inference completed') return dataset diff --git a/external/mmdetection/submodule b/external/mmdetection/submodule index 628ce1b6f37..6adb9077d24 160000 --- a/external/mmdetection/submodule +++ b/external/mmdetection/submodule @@ -1 +1 @@ -Subproject commit 628ce1b6f37150428be3c93445147d7c9c289456 +Subproject commit 6adb9077d24ca323a928f1ebd92ca8443f0f9394 diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py index 9679a899f6c..9f0af1ad653 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py @@ -49,7 +49,7 @@ ) from mmseg.apis import export_model -from mmseg.core.hooks.auxiliary_hooks import FeatureVectorHook, SaliencyMapHook +from mmseg.core.hooks.auxiliary_hooks import FeatureVectorHook from mmseg.datasets import build_dataloader, build_dataset from mmseg.models import build_segmentor from mmseg.parallel import MMDataCPU @@ -196,16 +196,15 @@ def hook(module, input, output): pre_hook_handle = self._model.register_forward_pre_hook(pre_hook) hook_handle = self._model.register_forward_hook(hook) - prediction_results = self._infer_segmentor(self._model, self._config, dataset, dump_features=True, - dump_saliency_map=not is_evaluation) - self._add_predictions_to_dataset(prediction_results, dataset) + prediction_results = self._infer_segmentor(self._model, self._config, dataset, dump_features=True) + self._add_predictions_to_dataset(prediction_results, dataset, dump_soft_prediction=not is_evaluation) pre_hook_handle.remove() hook_handle.remove() return dataset - def _add_predictions_to_dataset(self, prediction_results, dataset): - for dataset_item, (prediction, feature_vector, saliency_map) in zip(dataset, prediction_results): + def _add_predictions_to_dataset(self, prediction_results, dataset, dump_soft_prediction): + for dataset_item, (prediction, feature_vector) in zip(dataset, prediction_results): soft_prediction = np.transpose(prediction, axes=(1, 2, 0)) hard_prediction = create_hard_prediction_from_soft_prediction( soft_prediction=soft_prediction, @@ -223,18 +222,26 @@ def _add_predictions_to_dataset(self, prediction_results, dataset): active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) dataset_item.append_metadata_item(active_score, model=self._task_environment.model) - if saliency_map is not None: - class_act_map = get_activation_map(saliency_map, (dataset_item.width, dataset_item.height)) - result_media = ResultMediaEntity(name="saliency_map", - type="Saliency map", - annotation_scene=dataset_item.annotation_scene, - roi=dataset_item.roi, - numpy=class_act_map) - dataset_item.append_metadata_item(result_media, model=self._task_environment.model) - - def _infer_segmentor(self, - model: torch.nn.Module, config: Config, dataset: DatasetEntity, - dump_features: bool = False, dump_saliency_map: bool = False) -> None: + if dump_soft_prediction: + for label_index, label in self._label_dictionary.items(): + if label_index == 0: + continue + if len(soft_prediction.shape) == 3: + current_label_soft_prediction = soft_prediction[:, :, label_index] + else: + current_label_soft_prediction = soft_prediction + + class_act_map = get_activation_map(current_label_soft_prediction) + result_media = ResultMediaEntity(name='Soft Prediction', + type='soft_prediction', + label=label, + annotation_scene=dataset_item.annotation_scene, + roi=dataset_item.roi, + numpy=class_act_map) + dataset_item.append_metadata_item(result_media, model=self._task_environment.model) + + def _infer_segmentor(self, model: torch.nn.Module, config: Config, dataset: DatasetEntity, + dump_features: bool = False) -> None: model.eval() test_config = prepare_for_testing(config, dataset) @@ -254,21 +261,18 @@ def _infer_segmentor(self, eval_predictions = [] feature_vectors = [] - saliency_maps = [] # Use a single gpu for testing. Set in both mm_val_dataloader and eval_model with FeatureVectorHook(model.module.backbone) if dump_features else nullcontext() as fhook: - with SaliencyMapHook(model.module.backbone) if dump_saliency_map else nullcontext() as shook: - for data in mm_val_dataloader: - with torch.no_grad(): - result = model(return_loss=False, output_logits=True, **data) - eval_predictions.extend(result) - feature_vectors = fhook.records if dump_features else [None] * len(dataset) - saliency_maps = shook.records if dump_saliency_map else [None] * len(dataset) - assert len(eval_predictions) == len(feature_vectors) == len(saliency_maps), \ + for data in mm_val_dataloader: + with torch.no_grad(): + result = model(return_loss=False, output_logits=True, **data) + eval_predictions.extend(result) + feature_vectors = fhook.records if dump_features else [None] * len(dataset) + assert len(eval_predictions) == len(feature_vectors), \ 'Number of elements should be the same, however, number of outputs are ' \ - f"{len(eval_predictions)}, {len(feature_vectors)}, and {len(saliency_maps)}" - predictions = zip(eval_predictions, feature_vectors, saliency_maps) + f"{len(eval_predictions)} and {len(feature_vectors)}" + predictions = zip(eval_predictions, feature_vectors) return predictions @check_input_parameters_type() diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/model_wrappers/blur.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/model_wrappers/blur.py index 5e4fde00296..82431788ace 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/model_wrappers/blur.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/model_wrappers/blur.py @@ -26,13 +26,16 @@ @check_input_parameters_type() -def get_actmap( - features: Union[np.ndarray, Iterable, int, float], output_res: Union[tuple, list] -): - am = cv2.resize(features, output_res) - am = cv2.applyColorMap(am, cv2.COLORMAP_JET) - am = cv2.cvtColor(am, cv2.COLOR_BGR2RGB) - return am +def get_activation_map(features: Union[np.ndarray, Iterable, int, float]): + min_soft_score = np.min(features) + max_soft_score = np.max(features) + factor = 255.0 / (max_soft_score - min_soft_score + 1e-12) + + float_act_map = factor * (features - min_soft_score) + int_act_map = np.uint8(np.floor(float_act_map)) + int_act_map = cv2.applyColorMap(int_act_map, cv2.COLORMAP_JET) + int_act_map = cv2.cvtColor(int_act_map, cv2.COLOR_BGR2RGB) + return int_act_map class BlurSegmentation(SegmentationModel): @@ -78,18 +81,16 @@ def postprocess(self, outputs: Dict[str, np.ndarray], metadata: Dict[str, Any]): soft_threshold=self.soft_threshold, blur_strength=self.blur_strength ) - hard_prediction = cv2.resize(hard_prediction, metadata['original_shape'][1::-1], 0, 0, interpolation=cv2.INTER_NEAREST) - - if 'feature_vector' not in outputs or 'saliency_map' not in outputs: - warnings.warn('Could not find Feature Vector and Saliency Map in OpenVINO output. ' - 'Please rerun OpenVINO export or retrain the model.') - metadata["saliency_map"] = None + hard_prediction = cv2.resize(hard_prediction, metadata['original_shape'][1::-1], 0, 0, + interpolation=cv2.INTER_NEAREST) + soft_prediction = cv2.resize(soft_prediction, metadata['original_shape'][1::-1], 0, 0, + interpolation=cv2.INTER_NEAREST) + metadata['soft_prediction'] = soft_prediction + + if 'feature_vector' not in outputs: + warnings.warn('Could not find Feature Vector in OpenVINO output. Please rerun export or retrain the model.') metadata["feature_vector"] = None else: - metadata["saliency_map"] = get_actmap( - outputs["saliency_map"][0], - (metadata["original_shape"][1], metadata["original_shape"][0]), - ) metadata["feature_vector"] = outputs["feature_vector"].reshape(-1) return hard_prediction diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py index 5ff7e6063a5..34e4ab085e8 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py @@ -105,11 +105,11 @@ def pre_process(self, image: np.ndarray) -> Tuple[Dict[str, np.ndarray], Dict[st @check_input_parameters_type() def post_process(self, prediction: Dict[str, np.ndarray], metadata: Dict[str, Any]) -> AnnotationSceneEntity: hard_prediction = self.model.postprocess(prediction, metadata) + soft_prediction = metadata['soft_prediction'] feature_vector = metadata['feature_vector'] - saliency_map = metadata['saliency_map'] predicted_scene = self.converter.convert_to_annotation(hard_prediction, metadata) - return predicted_scene, feature_vector, saliency_map + return predicted_scene, feature_vector, soft_prediction @check_input_parameters_type() def forward(self, inputs: Dict[str, np.ndarray]) -> Dict[str, np.ndarray]: @@ -164,25 +164,36 @@ def infer(self, inference_parameters: Optional[InferenceParameters] = None) -> DatasetEntity: if inference_parameters is not None: update_progress_callback = inference_parameters.update_progress - dump_saliency_map = not inference_parameters.is_evaluation + dump_soft_prediction = not inference_parameters.is_evaluation else: update_progress_callback = default_progress_callback - dump_saliency_map = True + dump_soft_prediction = True dataset_size = len(dataset) for i, dataset_item in enumerate(dataset, 1): - predicted_scene, feature_vector, saliency_map = self.inferencer.predict(dataset_item.numpy) + predicted_scene, feature_vector, soft_prediction = self.inferencer.predict(dataset_item.numpy) dataset_item.append_annotations(predicted_scene.annotations) if feature_vector is not None: feature_vector_media = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) dataset_item.append_metadata_item(feature_vector_media, model=self.model) - if dump_saliency_map and saliency_map is not None: - saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map", - annotation_scene=dataset_item.annotation_scene, - numpy=saliency_map, roi=dataset_item.roi) - dataset_item.append_metadata_item(saliency_map_media, model=self.model) + if dump_soft_prediction: + for label_index, label in self._label_dictionary.items(): + if label_index == 0: + continue + if len(soft_prediction.shape) == 3: + current_label_soft_prediction = soft_prediction[:, :, label_index] + else: + current_label_soft_prediction = soft_prediction + class_act_map = get_activation_map(current_label_soft_prediction) + result_media = ResultMediaEntity(name=f'{label.name}', + type='Soft Prediction', + label=label, + annotation_scene=dataset_item.annotation_scene, + roi=dataset_item.roi, + numpy=class_act_map) + dataset_item.append_metadata_item(result_media, model=self.model) update_progress_callback(int(i / dataset_size * 100)) diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/ote_utils.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/ote_utils.py index 9250d7bc819..e567c290658 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/ote_utils.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/ote_utils.py @@ -44,11 +44,16 @@ def get_task_class(path: str): @check_input_parameters_type() -def get_activation_map(features: Union[np.ndarray, Iterable, int, float], output_res: Union[tuple, list]): - am = cv2.resize(features, output_res) - am = cv2.applyColorMap(am, cv2.COLORMAP_JET) - am = cv2.cvtColor(am, cv2.COLOR_BGR2RGB) - return am +def get_activation_map(features: Union[np.ndarray, Iterable, int, float]): + min_soft_score = np.min(features) + max_soft_score = np.max(features) + factor = 255.0 / (max_soft_score - min_soft_score + 1e-12) + + float_act_map = factor * (features - min_soft_score) + int_act_map = np.uint8(np.floor(float_act_map)) + int_act_map = cv2.applyColorMap(int_act_map, cv2.COLORMAP_JET) + int_act_map = cv2.cvtColor(int_act_map, cv2.COLOR_BGR2RGB) + return int_act_map class TrainingProgressCallback(TimeMonitorCallback): diff --git a/external/mmsegmentation/submodule b/external/mmsegmentation/submodule index 337c109596a..2f4c77520e8 160000 --- a/external/mmsegmentation/submodule +++ b/external/mmsegmentation/submodule @@ -1 +1 @@ -Subproject commit 337c109596acf7f06a73c199f368a8f0f18d026c +Subproject commit 2f4c77520e8ec5f0618edfbd0e9793be7d3d8a73 diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py index 8b5bffec63f..0ee5f73b840 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py @@ -17,7 +17,6 @@ from mpa.utils.logger import get_logger from mpa_tasks.apis import BaseTask, TrainType from mpa_tasks.apis.classification import ClassificationConfig -from mpa_tasks.utils.data_utils import get_actmap from ote_sdk.configuration import cfg_helper from ote_sdk.configuration.helper.utils import ids_to_strings from ote_sdk.entities.datasets import DatasetEntity @@ -60,6 +59,7 @@ from ote_sdk.usecases.tasks.interfaces.unload_interface import IUnload from ote_sdk.utils.argument_checks import check_input_parameters_type from ote_sdk.utils.labels_utils import get_empty_label +from ote_sdk.utils.vis_utils import get_actmap from torchreid_tasks.nncf_task import OTEClassificationNNCFTask # from torchreid_tasks.utils import TrainingProgressCallback @@ -246,7 +246,7 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, update_progre if saliency_map is not None: saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) - saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map", + saliency_map_media = ResultMediaEntity(name="Saliency Map", type="saliency_map", annotation_scene=dataset_item.annotation_scene, numpy=saliency_map, roi=dataset_item.roi, label=item_labels[0].label) diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py index 1d647c9d851..78265ec08f4 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/detection/task.py @@ -56,6 +56,8 @@ from detection_tasks.apis.detection import OTEDetectionNNCFTask from ote_sdk.utils.argument_checks import check_input_parameters_type from ote_sdk.entities.model_template import parse_model_template +from ote_sdk.utils.vis_utils import get_actmap + logger = get_logger() @@ -265,9 +267,8 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, confidence_th dataset_item.append_metadata_item(active_score, model=self._task_environment.model) if saliency_map is not None: - width, height = dataset_item.width, dataset_item.height - saliency_map = cv2.resize(saliency_map, (width, height), interpolation=cv2.INTER_NEAREST) - saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map", + saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height)) + saliency_map_media = ResultMediaEntity(name="Saliency Map", type="saliency_map", annotation_scene=dataset_item.annotation_scene, numpy=saliency_map, roi=dataset_item.roi) dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py index 0e6e4ccec7b..6b25e9528b9 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py @@ -11,12 +11,12 @@ import torch from mmcv.utils import ConfigDict from segmentation_tasks.apis.segmentation.config_utils import remove_from_config -from segmentation_tasks.apis.segmentation.ote_utils import TrainingProgressCallback, InferenceProgressCallback +from segmentation_tasks.apis.segmentation.ote_utils import (TrainingProgressCallback, InferenceProgressCallback, + get_activation_map) from segmentation_tasks.extension.utils.hooks import OTELoggerHook from mpa import MPAConstants from mpa_tasks.apis import BaseTask, TrainType from mpa_tasks.apis.segmentation import SegmentationConfig -from mpa_tasks.utils.data_utils import get_actmap from mpa.utils.config_utils import MPAConfig from mpa.utils.logger import get_logger from ote_sdk.configuration import cfg_helper @@ -73,7 +73,6 @@ def infer(self, ) -> DatasetEntity: logger.info('infer()') dump_features = True - dump_saliency_map = not inference_parameters.is_evaluation if inference_parameters else True if inference_parameters is not None: update_progress_callback = inference_parameters.update_progress @@ -87,13 +86,11 @@ def infer(self, stage_module = 'SegInferrer' self._data_cfg = self._init_test_data_cfg(dataset) self._label_dictionary = dict(enumerate(self._labels, 1)) - results = self._run_task(stage_module, mode='train', dataset=dataset, dump_features=dump_features, - dump_saliency_map=dump_saliency_map) + results = self._run_task(stage_module, mode='train', dataset=dataset, dump_features=dump_features) logger.debug(f'result of run_task {stage_module} module = {results}') predictions = results['outputs'] - prediction_results = zip(predictions['eval_predictions'], predictions['feature_vectors'], - predictions['saliency_maps']) - self._add_predictions_to_dataset(prediction_results, dataset, dump_saliency_map=not is_evaluation) + prediction_results = zip(predictions['eval_predictions'], predictions['feature_vectors']) + self._add_predictions_to_dataset(prediction_results, dataset, dump_soft_prediction=not is_evaluation) return dataset def evaluate(self, @@ -211,10 +208,10 @@ def _init_test_data_cfg(self, dataset: DatasetEntity): ) return data_cfg - def _add_predictions_to_dataset(self, prediction_results, dataset, dump_saliency_map): + def _add_predictions_to_dataset(self, prediction_results, dataset, dump_soft_prediction): """ Loop over dataset again to assign predictions. Convert from MMSegmentation format to OTE format. """ - for dataset_item, (prediction, feature_vector, saliency_map) in zip(dataset, prediction_results): + for dataset_item, (prediction, feature_vector) in zip(dataset, prediction_results): soft_prediction = np.transpose(prediction[0], axes=(1, 2, 0)) hard_prediction = create_hard_prediction_from_soft_prediction( soft_prediction=soft_prediction, @@ -232,12 +229,23 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, dump_saliency active_score = TensorEntity(name="representation_vector", numpy=feature_vector.reshape(-1)) dataset_item.append_metadata_item(active_score, model=self._task_environment.model) - if dump_saliency_map and saliency_map is not None: - saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height) ) - saliency_map_media = ResultMediaEntity(name="saliency_map", type="Saliency map", - annotation_scene=dataset_item.annotation_scene, - numpy=saliency_map, roi=dataset_item.roi) - dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model) + if dump_soft_prediction: + for label_index, label in self._label_dictionary.items(): + if label_index == 0: + continue + if len(soft_prediction.shape) == 3: + current_label_soft_prediction = soft_prediction[:, :, label_index] + else: + current_label_soft_prediction = soft_prediction + + class_act_map = get_activation_map(current_label_soft_prediction) + result_media = ResultMediaEntity(name='Soft Prediction', + type='soft_prediction', + label=label, + annotation_scene=dataset_item.annotation_scene, + roi=dataset_item.roi, + numpy=class_act_map) + dataset_item.append_metadata_item(result_media, model=self._task_environment.model) @staticmethod def _patch_datasets(config: MPAConfig, domain=Domain.SEGMENTATION): diff --git a/external/model-preparation-algorithm/mpa_tasks/utils/data_utils.py b/external/model-preparation-algorithm/mpa_tasks/utils/data_utils.py index 5545f6d1c99..b820e5762e3 100644 --- a/external/model-preparation-algorithm/mpa_tasks/utils/data_utils.py +++ b/external/model-preparation-algorithm/mpa_tasks/utils/data_utils.py @@ -2,11 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # -from typing import Union, Iterable - -import cv2 -import numpy as np - from mpa.utils.logger import get_logger logger = get_logger() @@ -33,11 +28,3 @@ def get_old_new_img_indices(labels, new_classes, dataset): else: ids_old.append(i) return {'old': ids_old, 'new': ids_new} - - -def get_actmap(saliency_map: Union[np.ndarray, Iterable, int, float], - output_res: Union[tuple, list]): - saliency_map = cv2.resize(saliency_map, output_res) - saliency_map = cv2.applyColorMap(saliency_map, cv2.COLORMAP_JET) - saliency_map = cv2.cvtColor(saliency_map, cv2.COLOR_BGR2RGB) - return saliency_map diff --git a/external/model-preparation-algorithm/submodule b/external/model-preparation-algorithm/submodule index 554cd30f60f..e8874ea439b 160000 --- a/external/model-preparation-algorithm/submodule +++ b/external/model-preparation-algorithm/submodule @@ -1 +1 @@ -Subproject commit 554cd30f60f53c7789f099ed1f9f53626d460f6d +Subproject commit e8874ea439bcf066de9cf0dd22d6fb8805be741c diff --git a/ote_sdk/ote_sdk/utils/vis_utils.py b/ote_sdk/ote_sdk/utils/vis_utils.py new file mode 100644 index 00000000000..98f6e238550 --- /dev/null +++ b/ote_sdk/ote_sdk/utils/vis_utils.py @@ -0,0 +1,26 @@ +# Copyright (C) 2021-2022 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +from typing import Union, Iterable + +import cv2 +import numpy as np + + +def get_actmap(saliency_map: Union[np.ndarray, Iterable, int, float], output_res: Union[tuple, list]) -> np.ndarray: + """ Get activation map (heatmap) from saliency map + + Args: + saliency_map (Union[np.ndarray, Iterable, int, float]): Saliency map with pixel values from 0-255 + output_res (Union[tuple, list]): Output resolution + + Returns: + np.ndarray: activation map (heatmap) + """ + if len(saliency_map.shape) == 3: + saliency_map = saliency_map[0] + + saliency_map = cv2.resize(saliency_map, output_res) + saliency_map = cv2.applyColorMap(saliency_map, cv2.COLORMAP_JET) + saliency_map = cv2.cvtColor(saliency_map, cv2.COLOR_BGR2RGB) + return saliency_map From f18bf97170c65291ffda5a8a765b5317e62259ad Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Tue, 30 Aug 2022 12:03:28 +0100 Subject: [PATCH 2/5] fix inconsistence naming --- .../segmentation_tasks/apis/segmentation/openvino_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py index 34e4ab085e8..7ec918eaeb0 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py @@ -187,8 +187,8 @@ def infer(self, else: current_label_soft_prediction = soft_prediction class_act_map = get_activation_map(current_label_soft_prediction) - result_media = ResultMediaEntity(name=f'{label.name}', - type='Soft Prediction', + result_media = ResultMediaEntity(name='Soft Prediction', + type='soft_prediction', label=label, annotation_scene=dataset_item.annotation_scene, roi=dataset_item.roi, From acffde92c63f46765dc1c9141cebabea36c6187b Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Wed, 31 Aug 2022 08:48:22 +0100 Subject: [PATCH 3/5] update submodule --- external/mmdetection/submodule | 2 +- external/mmsegmentation/submodule | 2 +- external/model-preparation-algorithm/submodule | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/external/mmdetection/submodule b/external/mmdetection/submodule index 6adb9077d24..258ca9e6a35 160000 --- a/external/mmdetection/submodule +++ b/external/mmdetection/submodule @@ -1 +1 @@ -Subproject commit 6adb9077d24ca323a928f1ebd92ca8443f0f9394 +Subproject commit 258ca9e6a35a66495cd4f80a65394dac561cac88 diff --git a/external/mmsegmentation/submodule b/external/mmsegmentation/submodule index 2f4c77520e8..c652095c2ff 160000 --- a/external/mmsegmentation/submodule +++ b/external/mmsegmentation/submodule @@ -1 +1 @@ -Subproject commit 2f4c77520e8ec5f0618edfbd0e9793be7d3d8a73 +Subproject commit c652095c2ff342561bf9c9bdf8567522cf11d19a diff --git a/external/model-preparation-algorithm/submodule b/external/model-preparation-algorithm/submodule index e8874ea439b..484f18f531e 160000 --- a/external/model-preparation-algorithm/submodule +++ b/external/model-preparation-algorithm/submodule @@ -1 +1 @@ -Subproject commit e8874ea439bcf066de9cf0dd22d6fb8805be741c +Subproject commit 484f18f531ecf0987d62bc2c6890f4aa8007b206 From 59e94dde8a670a602b1014daca26fe57ebc11013 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Wed, 31 Aug 2022 19:13:19 +0100 Subject: [PATCH 4/5] remove redundant check --- .../segmentation_tasks/apis/segmentation/inference_task.py | 5 +---- .../mpa_tasks/apis/segmentation/task.py | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py index 9f0af1ad653..02431bdfc74 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/inference_task.py @@ -226,10 +226,7 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, dump_soft_pre for label_index, label in self._label_dictionary.items(): if label_index == 0: continue - if len(soft_prediction.shape) == 3: - current_label_soft_prediction = soft_prediction[:, :, label_index] - else: - current_label_soft_prediction = soft_prediction + current_label_soft_prediction = soft_prediction[:, :, label_index] class_act_map = get_activation_map(current_label_soft_prediction) result_media = ResultMediaEntity(name='Soft Prediction', diff --git a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py index 6b25e9528b9..49362a711cb 100644 --- a/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py +++ b/external/model-preparation-algorithm/mpa_tasks/apis/segmentation/task.py @@ -233,11 +233,7 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, dump_soft_pre for label_index, label in self._label_dictionary.items(): if label_index == 0: continue - if len(soft_prediction.shape) == 3: - current_label_soft_prediction = soft_prediction[:, :, label_index] - else: - current_label_soft_prediction = soft_prediction - + current_label_soft_prediction = soft_prediction[:, :, label_index] class_act_map = get_activation_map(current_label_soft_prediction) result_media = ResultMediaEntity(name='Soft Prediction', type='soft_prediction', From fd3c87584ef36ff7dfcd24bf3aca20b780201c58 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Wed, 31 Aug 2022 19:17:25 +0100 Subject: [PATCH 5/5] update --- .../segmentation_tasks/apis/segmentation/openvino_task.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py b/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py index 7ec918eaeb0..b2695cb2f10 100644 --- a/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py +++ b/external/mmsegmentation/segmentation_tasks/apis/segmentation/openvino_task.py @@ -182,10 +182,7 @@ def infer(self, for label_index, label in self._label_dictionary.items(): if label_index == 0: continue - if len(soft_prediction.shape) == 3: - current_label_soft_prediction = soft_prediction[:, :, label_index] - else: - current_label_soft_prediction = soft_prediction + current_label_soft_prediction = soft_prediction[:, :, label_index] class_act_map = get_activation_map(current_label_soft_prediction) result_media = ResultMediaEntity(name='Soft Prediction', type='soft_prediction',