Replace current saliency map generation with Recipro-CAM for cls

external/deep-object-reid/torchreid_tasks/model_wrappers/classification.py

@@ -91,7 +91,7 @@ def postprocess(self, outputs: Dict[str, np.ndarray], metadata: Dict[str, Any]):
 
     @check_input_parameters_type()
     def postprocess_aux_outputs(self, outputs: Dict[str, np.ndarray], metadata: Dict[str, Any]):
-        actmap = get_actmap(outputs['saliency_map'][0], (metadata['original_shape'][1], metadata['original_shape'][0]))
+        saliency_map = outputs['saliency_map'][0]
         repr_vector = outputs['feature_vector'].reshape(-1)
 
         logits = outputs[self.out_layer_name].squeeze()
@@ -103,16 +103,7 @@ def postprocess_aux_outputs(self, outputs: Dict[str, np.ndarray], metadata: Dict
 
         act_score = float(np.max(probs) - np.min(probs))
 
-        return actmap, repr_vector, act_score
-
-
-@check_input_parameters_type()
-def get_actmap(features: Union[np.ndarray, Iterable, int, float],
-               output_res: Union[tuple, list]):
-    am = cv2.resize(features, output_res)
-    am = cv2.applyColorMap(am, cv2.COLORMAP_JET)
-    am = cv2.cvtColor(am, cv2.COLOR_BGR2RGB)
-    return am
+        return saliency_map, repr_vector, act_score
 
 
 @check_input_parameters_type()

external/deep-object-reid/torchreid_tasks/openvino_task.py

@@ -70,7 +70,7 @@
     import warnings
     warnings.warn("ModelAPI was not found.")
 from torchreid_tasks.parameters import OTEClassificationParameters
-from torchreid_tasks.utils import get_multihead_class_info
+from torchreid_tasks.utils import get_multihead_class_info, get_actmap
 
 from zipfile import ZipFile
 
@@ -183,19 +183,36 @@ def infer(self, dataset: DatasetEntity,
             dump_features = not inference_parameters.is_evaluation
         dataset_size = len(dataset)
         for i, dataset_item in enumerate(dataset, 1):
-            predicted_scene, actmap, repr_vector, act_score = self.inferencer.predict(dataset_item.numpy)
+            predicted_scene, saliency_map, repr_vector, act_score = self.inferencer.predict(dataset_item.numpy)
             dataset_item.append_labels(predicted_scene.annotations[0].get_labels())
             active_score_media = FloatMetadata(name="active_score", value=act_score,
                                                float_type=FloatType.ACTIVE_SCORE)
             dataset_item.append_metadata_item(active_score_media, model=self.model)
             feature_vec_media = TensorEntity(name="representation_vector", numpy=repr_vector.reshape(-1))
             dataset_item.append_metadata_item(feature_vec_media, model=self.model)
             if dump_features:
-                saliency_media = ResultMediaEntity(name="Saliency Map", type="saliency_map",
-                                                   annotation_scene=dataset_item.annotation_scene,
-                                                   numpy=actmap, roi=dataset_item.roi,
-                                                   label=predicted_scene.annotations[0].get_labels()[0].label)
-                dataset_item.append_metadata_item(saliency_media, model=self.model)
+                if saliency_map.ndim == 2:
+                    # Single saliency map per image, support e.g. EigenCAM use case
+                    actmap = get_actmap(saliency_map, (dataset_item.width, dataset_item.height))
+                    saliency_media = ResultMediaEntity(name="Saliency Map", type="saliency_map",
+                                                       annotation_scene=dataset_item.annotation_scene,
+                                                       numpy=actmap, roi=dataset_item.roi,
+                                                       label=predicted_scene.annotations[0].get_labels()[0].label)
+                    dataset_item.append_metadata_item(saliency_media, model=self.model)
+                elif saliency_map.ndim == 3:
+                    # Multiple saliency maps per image (class-wise saliency map), support e.g. Recipro-CAM use case
+                    for class_id, class_wise_saliency_map in enumerate(saliency_map):
+                        actmap = get_actmap(class_wise_saliency_map, (dataset_item.width, dataset_item.height))
+                        class_name_str = self.task_environment.get_labels(include_empty=True)[class_id].name
+                        saliency_media = ResultMediaEntity(name=f"Saliency Map: {class_name_str}",
+                                                           type="saliency_map",
+                                                           annotation_scene=dataset_item.annotation_scene,
+                                                           numpy=actmap, roi=dataset_item.roi,
+                                                           label=predicted_scene.annotations[0].get_labels()[0].label)
+                        dataset_item.append_metadata_item(saliency_media, model=self.model)
+                else:
+                    raise RuntimeError(f'Single saliency map has to be 2 or 3-dimensional, '
+                                       f'but got {saliency_map.ndim} dims')
 
             update_progress_callback(int(i / dataset_size * 100))
         return dataset

external/deep-object-reid/torchreid_tasks/utils.py

@@ -626,3 +626,12 @@ def after_train_epoch(self, runner: BaseRunner):
         runner._iter -= 1
         super().after_train_epoch(runner)
         runner._iter += 1
+
+
+@check_input_parameters_type()
+def get_actmap(features: Union[np.ndarray, Iterable, int, float],
+               output_res: Union[tuple, list]):
+    am = cv.resize(features, output_res)
+    am = cv.applyColorMap(am, cv.COLORMAP_JET)
+    am = cv.cvtColor(am, cv.COLOR_BGR2RGB)
+    return am

external/model-preparation-algorithm/mpa_tasks/apis/classification/task.py

@@ -263,16 +263,38 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, update_progre
                 dataset_item.append_metadata_item(active_score, model=self._task_environment.model)
 
             if saliency_map is not None:
-                saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height))
-                saliency_map_media = ResultMediaEntity(
-                    name="Saliency Map",
-                    type="saliency_map",
-                    annotation_scene=dataset_item.annotation_scene,
-                    numpy=saliency_map,
-                    roi=dataset_item.roi,
-                    label=item_labels[0].label,
-                )
-                dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model)
+                if saliency_map.ndim == 2:
+                    # Single saliency map per image, support e.g. EigenCAM use case
+                    saliency_map = get_actmap(saliency_map, (dataset_item.width, dataset_item.height))
+                    saliency_map_media = ResultMediaEntity(
+                        name="Saliency Map",
+                        type="saliency_map",
+                        annotation_scene=dataset_item.annotation_scene,
+                        numpy=saliency_map,
+                        roi=dataset_item.roi,
+                        label=item_labels[0].label,
+                    )
+                    dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model)
+                elif saliency_map.ndim == 3:
+                    # Multiple saliency maps per image (class-wise saliency map), support e.g. Recipro-CAM use case
+                    for class_id, class_wise_saliency_map in enumerate(saliency_map):
+                        class_wise_saliency_map = get_actmap(
+                            class_wise_saliency_map, (dataset_item.width, dataset_item.height)
+                        )
+                        class_name_str = self._labels[class_id].name
+                        saliency_map_media = ResultMediaEntity(
+                            name=f"Saliency Map: {class_name_str}",
+                            type="saliency_map",
+                            annotation_scene=dataset_item.annotation_scene,
+                            numpy=class_wise_saliency_map,
+                            roi=dataset_item.roi,
+                            label=item_labels[0].label,
+                        )
+                        dataset_item.append_metadata_item(saliency_map_media, model=self._task_environment.model)
+                else:
+                    raise RuntimeError(
+                        f"Single saliency map has to be 2 or 3-dimensional, " f"but got {saliency_map.ndim} dims"
+                    )
 
             update_progress_callback(int(i / dataset_size * 100))
 

external/model-preparation-algorithm/tests/test_xai.py

@@ -0,0 +1,73 @@
+# Copyright (C) 2022 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#
+
+import numpy as np
+import torch
+
+from mmcls.models import build_classifier
+
+from mpa.modules.hooks.auxiliary_hooks import ReciproCAMHook
+
+
+torch.manual_seed(0)
+
+
+class TestExplainMethods:
+    @staticmethod
+    def get_model():
+        model_cfg = dict(
+            type="ImageClassifier",
+            backbone=dict(type="ResNet", depth=18, num_stages=4, out_indices=(3,), style="pytorch"),
+            neck=dict(type="GlobalAveragePooling"),
+            head=dict(
+                type="LinearClsHead",
+                num_classes=2,
+                in_channels=512,
+                loss=dict(type="CrossEntropyLoss", loss_weight=1.0),
+                topk=(1, 5),
+            ),
+        )
+
+        model = build_classifier(model_cfg)
+        return model.eval()
+
+    def test_recipro_cam(self):
+        model = self.get_model()
+        img = torch.rand(2, 3, 224, 224) - 0.5
+        data = {"img_metas": {}, "img": img}
+
+        with ReciproCAMHook(model) as rcam_hook:
+            _ = model(return_loss=False, **data)
+        saliency_maps = rcam_hook.records
+
+        assert len(saliency_maps) == 2
+        assert saliency_maps[0].ndim == 3
+        assert saliency_maps[0].shape == (2, 7, 7)
+
+        sal_class0_reference = np.array(
+            [
+                [27, 112, 105, 92, 114, 84, 73],
+                [0, 88, 144, 91, 58, 103, 96],
+                [54, 155, 121, 105, 1, 111, 146],
+                [34, 199, 122, 159, 126, 164, 83],
+                [59, 70, 237, 149, 127, 164, 148],
+                [14, 213, 150, 135, 124, 215, 43],
+                [36, 98, 114, 61, 99, 255, 30],
+            ],
+            dtype=np.uint8,
+        )
+        sal_class1_reference = np.array(
+            [
+                [227, 142, 149, 162, 140, 170, 181],
+                [255, 166, 110, 163, 196, 151, 158],
+                [200, 99, 133, 149, 253, 142, 108],
+                [220, 55, 132, 95, 128, 90, 171],
+                [195, 184, 17, 105, 127, 90, 106],
+                [240, 41, 104, 119, 130, 39, 211],
+                [218, 156, 140, 193, 155, 0, 224],
+            ],
+            dtype=np.uint8,
+        )
+        assert (saliency_maps[0][0] == sal_class0_reference).all()
+        assert (saliency_maps[0][1] == sal_class1_reference).all()