Merge branch 'feature/classification-history' into devel

.travis.yml

@@ -1,3 +1,4 @@
+
 sudo: required
 language: python
 

gui/threads/thermo_thread.py

@@ -88,7 +88,7 @@ def run(self):
                     self.msleep(self.pause_time)
 
                 Logger.debug("Using video frame {}".format(frame_id))
-                # Perfom one step in the input video (i.e. analyze one frame)
+                # Perform one step in the input video (i.e. analyze one frame)
                 self.app.step(frame_id, frame)
                 # Perform inference (classification on the detected modules)
                 self.app.classify_detected_modules()

thermography/module_map.py

@@ -17,6 +17,7 @@ def __init__(self, ID: int, rectangle: np.ndarray, frame_id: int):
             self.rectangle_history = {}
 
             self.cumulated_motion = np.array([0, 0], dtype=np.float32)
+            self.__all_probabilities = []
 
             self.add(rectangle, frame_id)
 
@@ -48,6 +49,19 @@ def add_motion(self, frame_id: int, motion_estimate: np.ndarray):
             if frame_id != self.frame_id_history[-1]:
                 self.cumulated_motion += motion_estimate
 
+        def update_probability(self, prob: np.ndarray) -> None:
+            """
+            Updates the current probability distribution over the class labels of this module.
+            :param prob: A 1D numpy array of size 'num_classes' representing the classification probability.
+            """
+            self.__all_probabilities.append(prob)
+
+        @property
+        def mean_probability(self):
+            if len(self.__all_probabilities) == 0:
+                raise RuntimeError("No probabilities assigned to current module {}".format(self.ID))
+            return np.mean(self.__all_probabilities, axis=0)
+
     def __init__(self):
         Logger.debug("Creating the module map")
         # A dictionary of modules and their centers keyed by their ID.
@@ -92,7 +106,7 @@ def insert(self, rectangle_list: list, frame_id: int, motion_estimate: np.ndarra
                 # Compute the ID of the rectangle in the global map which is most similar to the current rectangle.
                 # This computation involves the evaluation of the surface between the corresponding edges of the
                 # rectangles of interest.
-                most_similar_ID = self.__find_most_similar_module(rectangle, area_threshold_ratio=0.2)
+                most_similar_ID = self.__find_most_similar_module(rectangle, area_threshold_ratio=0.5)
 
                 if most_similar_ID is None:
                     associations.append(None)
@@ -117,6 +131,15 @@ def insert(self, rectangle_list: list, frame_id: int, motion_estimate: np.ndarra
 
         self.__store_old_modules(frame_id)
 
+    def update_class_belief(self, probabilities: dict) -> None:
+        """
+        Updates the current class probability for the modules being detected in the last step.
+
+        :param probabilities: A dictionary keyed by the module ID, whose value is a probability distribution over the classes.
+        """
+        for module_id, prob in probabilities.items():
+            self.global_module_map[module_id].update_probability(prob)
+
     def __find_most_similar_module(self, rectangle: np.ndarray, area_threshold_ratio: float) -> int:
         """
         Finds the most similar rectangle in the global map by computing the surface between each rectangle stored in the

thermography/thermo_app.py

@@ -101,32 +101,32 @@ def create_segment_image(self):
     def create_rectangle_image(self):
         Logger.debug("Creating rectangle image")
         base_image = self.last_scaled_frame_rgb.copy()
-        if self.last_rectangles is not None and len(self.last_rectangles) > 0:
-            mean_color = np.mean(base_image, axis=(0, 1))
-            mask = np.zeros_like(base_image)
-            if mean_color[0] == mean_color[1] == mean_color[2]:
-                mean_color = np.array([255, 255, 0])
-            opposite_color = np.array([255, 255, 255]) - mean_color
-            opposite_color = (int(opposite_color[0]), int(opposite_color[1]), int(opposite_color[2]))
-            for rectangle in self.last_rectangles:
-                cv2.polylines(base_image, np.int32([rectangle]), True, opposite_color, 1, cv2.LINE_AA)
-                cv2.fillConvexPoly(mask, np.int32([rectangle]), (255, 0, 0), cv2.LINE_4)
-
-            cv2.addWeighted(base_image, 1.0, mask, 0.8, 0, base_image)
+        mask = np.zeros_like(base_image)
+
+        for module_id, module in self.module_map.global_module_map.items():
+            if module.frame_id_history[-1] == self.last_frame_id:
+
+                module_coords = module.last_rectangle - np.int32(module.cumulated_motion)
+                mean_prob = module.mean_probability
+                color = color_from_probabilities(mean_prob)
+
+                cv2.polylines(base_image, np.int32([module_coords]), True, color, 1, cv2.LINE_AA)
+                cv2.fillConvexPoly(mask, np.int32([module_coords]), color, cv2.LINE_4)
+            else:
+                continue
+
+        cv2.addWeighted(base_image, 1.0, mask, 0.4, 0, base_image)
         return base_image
 
     def create_classes_image(self):
         Logger.debug("Creating classes image")
         base_image = self.last_scaled_frame_rgb.copy()
-        working_color = np.array([0, 255, 0])
-        broken_color = np.array([0, 0, 255])
-        misdetected_color = np.array([255, 0, 0])
-        for module_id, prob in self.last_probabilities.items():
-            module = self.module_map.global_module_map[module_id]
+
+        for module_id, module in self.module_map.global_module_map.items():
             module_coords = module.last_rectangle - np.int32(module.cumulated_motion)
             module_center = module.last_center - np.int32(module.cumulated_motion)
-            color = prob[0] * working_color + prob[1] * broken_color + prob[2] * misdetected_color
-            color = (int(color[0]), int(color[1]), int(color[2]))
+            mean_prob = module.mean_probability
+            color = color_from_probabilities(mean_prob)
 
             cv2.circle(base_image, (int(module_center[0]), int(module_center[1])), 6, color, cv2.FILLED, cv2.LINE_AA)
             cv2.polylines(base_image, np.int32([module_coords]), True, color, 1, cv2.LINE_AA)
@@ -262,6 +262,8 @@ def classify_detected_modules(self):
         for module, prob in zip(module_list, probabilities):
             self.last_probabilities[module["id"]] = prob
 
+        self.module_map.update_class_belief(self.last_probabilities)
+
     def step(self, frame_id, frame):
         self.last_frame_id = frame_id
         self.last_input_frame = frame

thermography/utils/display.py

@@ -2,7 +2,7 @@
 import numpy as np
 
 __all__ = ["draw_intersections", "draw_motion", "draw_rectangles", "draw_segments",
-           "random_color"]
+           "random_color", "color_from_probabilities"]
 
 
 def draw_intersections(intersections: list, base_image: np.ndarray, windows_name: str):
@@ -126,3 +126,14 @@ def random_color() -> tuple:
     """
     c = np.random.randint(0, 255, 3)
     return int(c[0]), int(c[1]), int(c[2])
+
+
+def color_from_probabilities(prob: np.ndarray) -> tuple:
+    """
+    Constructs a color tuple given the probability distribution prob.
+
+    :param prob: A three dimensional numpy array containing class probabilities.
+    :return: The color associated to the probability distribution.
+    """
+    color = np.diag(prob).dot(np.ones(shape=[3, 1]) * 255.0)
+    return (int(color[2]), int(color[0]), int(color[1]))