main.py

import os
import math
import glob
import json
import shutil
import argparse
import numpy as np

# import custom functions
from utils.is_float_between_0_and_1 import is_float_between_0_and_1
from utils.log_average_miss_rate import log_average_miss_rate
from utils.file_lines_to_list import file_lines_to_list
from utils.draw_text_in_image import draw_text_in_image
from utils.draw_plot_func import draw_plot_func
from utils.annotation_converters import RotatedBBoxConverter
from iou_rotate import iou_rotate_calculate
from utils.average_precision import voc_ap
from utils.error import error

MINOVERLAP = 0.3  # default value (defined in the PASCAL VOC2012 challenge)

parser = argparse.ArgumentParser()
parser.add_argument('-na', '--no-animation', help="no animation is shown.", action="store_true")
parser.add_argument('-np', '--no-plot', help="no plot is shown.", action="store_true")
parser.add_argument('-q', '--quiet', help="minimalistic console output.", action="store_true")
# argparse receiving list of classes to be ignored
parser.add_argument('-i', '--ignore', nargs='+', type=str, help="ignore a list of classes.")
# argparse receiving list of classes with specific IoU (e.g., python main.py --set-class-iou person 0.7)
parser.add_argument('--set-class-iou', nargs='+', type=str, help="set IoU for a specific class.")
parser.add_argument('--gpu', type=str, default='cpu', help="set gpu available")
# argparse receiving each of box mode.
parser.add_argument('--gt-box-mode', type=str, default='rotated', help="input the Ground Truth Box mode in [horizontal,rotated]")
parser.add_argument('--pred-box-mode', type=str, default='rotated', help="input the Predict Box mode in [horizontal,rotated]")
args = parser.parse_args()

# make sure that the cwd() is the location of the python script (so that every path makes sense)
os.chdir(os.path.dirname(os.path.abspath(__file__)))


def main(GT_PATH: os.path, DR_PATH: os.path, IMG_PATH: os.path):

    # if there are no classes to ignore then replace None by empty list
    if args.ignore is None:
        args.ignore = []

    specific_iou_flagged = False
    if args.set_class_iou is not None:
        specific_iou_flagged = True

    # Without image, no_animation is activate
    # if there are no images then no animation can be shown
    if os.path.exists(IMG_PATH):
        for dirpath, dirnames, files in os.walk(IMG_PATH):
            if not files:
                # no image files found
                args.no_animation = True
    else:
        args.no_animation = True

    # try to import OpenCV if the user didn't choose the option --no-animation
    show_animation = False
    if not args.no_animation:
        try:
            import cv2

            show_animation = True
        except ImportError:
            print("\"opencv-python\" not found, please install to visualize the results.")
            args.no_animation = True

    # try to import Matplotlib if the user didn't choose the option --no-plot
    draw_plot = False
    if not args.no_plot:
        try:
            import matplotlib.pyplot as plt
            draw_plot = True
        except ImportError:
            print("\"matplotlib\" not found, please install it to get the resulting plots.")
            args.no_plot = True

    """
     Create a ".temp_files/" and "results/" directory
    """

    TEMP_FILES_PATH = ".temp_files"
    if not os.path.exists(TEMP_FILES_PATH):  # if it doesn't exist already
        os.makedirs(TEMP_FILES_PATH)
    results_files_path = "mIoU-gt-" + args.gt_box_mode + "-pred-" + args.pred_box_mode + "-results"
    if os.path.exists(results_files_path):  # if it exist already
        # reset the results directory
        shutil.rmtree(results_files_path)

    os.makedirs(results_files_path)
    if draw_plot:
        os.makedirs(os.path.join(results_files_path, "classes"))
    if show_animation:
        os.makedirs(os.path.join(results_files_path, "images", "detections_one_by_one"))

    """
     ground-truth
         Load each of the ground-truth files into a temporary ".json" file.
         Create a list of all the class names present in the ground-truth (gt_classes).
    """
    # get a list with the ground-truth files
    ground_truth_files_list = glob.glob(GT_PATH + '/*.txt')
    if len(ground_truth_files_list) == 0:
        error("Error: No ground-truth files found!")
    ground_truth_files_list.sort()
    # dictionary with counter per class
    gt_counter_per_class = {}
    counter_images_per_class = {}

    for txt_file in ground_truth_files_list:
        file_id = txt_file.split(".txt", 1)[0]
        file_id = os.path.basename(os.path.normpath(file_id))
        # check if there is a correspondent rotated-horizontal-detection-results file
        temp_path = os.path.join(DR_PATH, (file_id + ".txt"))
        if not os.path.exists(temp_path):
            error_msg = "Error. File not found: {}\n".format(temp_path)
            error_msg += "(You can avoid this error message by running extra/intersect-gt-and-dr.py)"
            error(error_msg)
        lines_list = file_lines_to_list(txt_file)
        # create ground-truth dictionary
        bounding_boxes = []
        is_difficult = False
        already_seen_classes = []
        for line in lines_list:
            try:
                if "difficult" in line:
                    # Ground Truth information
                    class_name, cx, cy, width, height, theta, _difficult = line.split()
                    is_difficult = True
                else:
                    class_name, cx, cy, width, height, theta = line.split()
            except ValueError:
                error_msg = "Error: File " + txt_file + " in the wrong format.\n"
                error_msg += " Expected: <class_name> <left> <top> <right> <bottom> ['difficult']\n"
                error_msg += " Received: " + line
                error_msg += "\n\nIf you have a <class_name> with spaces between words you should remove them\n"
                error_msg += "by running the script \"remove_space.py\" or \"rename_class.py\" in the \"extra/\" folder."
                error(error_msg)
            # check if class is in the ignore list, if yes skip
            if class_name in args.ignore:
                continue
            bbox = cx + " " + cy + " " + width + " " + height + " " + theta
            if is_difficult:
                bounding_boxes.append({"class_name": class_name, "bbox": bbox, "used": False, "difficult": True})
                is_difficult = False
            else:
                bounding_boxes.append({"class_name": class_name, "bbox": bbox, "used": False})
                # count that object
                if class_name in gt_counter_per_class:
                    gt_counter_per_class[class_name] += 1
                else:
                    # if class didn't exist yet
                    gt_counter_per_class[class_name] = 1

                if class_name not in already_seen_classes:
                    if class_name in counter_images_per_class:
                        counter_images_per_class[class_name] += 1
                    else:
                        # if class didn't exist yet
                        counter_images_per_class[class_name] = 1
                    already_seen_classes.append(class_name)

        # dump bounding_boxes into a ".json" file
        with open(TEMP_FILES_PATH + "/" + file_id + "_ground_truth.json", 'w') as outfile:
            json.dump(bounding_boxes, outfile)

    gt_classes = list(gt_counter_per_class.keys())
    # let's sort the classes alphabetically
    gt_classes = sorted(gt_classes)
    n_classes = len(gt_classes)

    """
     Check format of the flag --set-class-iou (if used)
        e.g. check if class exists
    """
    if specific_iou_flagged:
        n_args = len(args.set_class_iou)
        error_msg = \
            '\n --set-class-iou [class_1] [IoU_1] [class_2] [IoU_2] [...]'
        if n_args % 2 != 0:
            error('Error, missing arguments. Flag usage:' + error_msg)
        specific_iou_classes = args.set_class_iou[::2]  # even
        iou_list = args.set_class_iou[1::2]  # odd
        if len(specific_iou_classes) != len(iou_list):
            error('Error, missing arguments. Flag usage:' + error_msg)
        for tmp_class in specific_iou_classes:
            if tmp_class not in gt_classes:
                error('Error, unknown class \"' + tmp_class + '\". Flag usage:' + error_msg)
        for num in iou_list:
            if not is_float_between_0_and_1(num):
                error('Error, IoU must be between 0.0 and 1.0. Flag usage:' + error_msg)

    """
     rotated-horizontal-detection-results
         Load each of the rotated-horizontal-detection-results files into a temporary ".json" file.
    """
    # get a list with the rotated-horizontal-detection-results files
    dr_files_list = glob.glob(DR_PATH + '/*.txt')
    dr_files_list.sort()

    for class_index, class_name in enumerate(gt_classes):
        bounding_boxes = []
        for txt_file in dr_files_list:
            # the first time it checks if all the corresponding ground-truth files exist
            file_id = txt_file.split(".txt", 1)[0]
            file_id = os.path.basename(os.path.normpath(file_id))
            temp_path = os.path.join(GT_PATH, (file_id + ".txt"))
            if class_index == 0:
                if not os.path.exists(temp_path):
                    error_msg = "Error. File not found: {}\n".format(temp_path)
                    error_msg += "(You can avoid this error message by running extra/intersect-gt-and-dr.py)"
                    error(error_msg)
            lines = file_lines_to_list(txt_file)
            for line in lines:
                try:
                    tmp_class_name, confidence, cx, cy, width, height, theta = line.split()
                except ValueError:
                    error_msg = "Error: File " + txt_file + " in the wrong format.\n"
                    error_msg += " Expected: <class_name> <confidence> <cx> <cy> <width> <height> <theta>\n"
                    error_msg += " Received: " + line
                    error(error_msg)
                if tmp_class_name == class_name:
                    bbox = cx + " " + cy + " " + width + " " + height + " " + theta
                    bounding_boxes.append({"confidence": confidence, "file_id": file_id, "bbox": bbox})

        # sort rotated-horizontal-detection-results by decreasing confidence
        bounding_boxes.sort(key=lambda x: float(x['confidence']), reverse=True)
        with open(TEMP_FILES_PATH + "/" + class_name + "_dr.json", 'w') as outfile:
            json.dump(bounding_boxes, outfile)

    """
     Calculate the AP for each class
    """

    sum_AP = 0.0
    sum_maIoU = 0.0
    ap_dictionary = {}
    lamr_dictionary = {}
    miou_dictionary = {}
    # open file to store the results

    with open(results_files_path + "/results.txt", 'w') as results_file:
        results_file.write("# AP and precision/recall per class\n")
        count_true_positives = {}
        i = 0
        for class_index, class_name in enumerate(gt_classes):
            count_true_positives[class_name] = 0
            """
             Load rotated-horizontal-detection-results of that class
            """
            dr_file = TEMP_FILES_PATH + "/" + class_name + "_dr.json"
            dr_data = json.load(open(dr_file))

            """
             Assign rotated-horizontal-detection-results to ground-truth objects
            """
            nd = len(dr_data)
            tp = [0] * nd  # creates an array of zeros of size nd
            fp = [0] * nd
            i = 0
            iou = []
            for idx, detection in enumerate(dr_data):
                file_id = detection["file_id"]
                # assign rotated-horizontal-detection-results to ground truth object if any
                # open ground-truth with that file_id
                gt_file = TEMP_FILES_PATH + "/" + file_id + "_ground_truth.json"
                ground_truth_data = json.load(open(gt_file))
                ovmax = -1
                """
                iou section(target of replace)
                """
                # # load detected object bounding-box
                rb_cvt = RotatedBBoxConverter()
                bb = [float(x) for x in detection["bbox"].split()]
                for obj in ground_truth_data:
                    if show_animation:
                        # find ground truth image
                        ground_truth_img = glob.glob1(IMG_PATH, file_id + ".*")
                        if len(ground_truth_img) == 0:
                            error("Error. Image not found with id: " + file_id)
                        elif len(ground_truth_img) > 1:
                            error("Error. Multiple image with id: " + file_id)
                        else:  # found image
                            # Load image
                            img = cv2.imread(IMG_PATH + "/" + ground_truth_img[0])
                            # load image with draws of multiple detections
                            img_cumulative_path = results_files_path + "/images/" + ground_truth_img[0]
                            if os.path.isfile(img_cumulative_path):
                                img_cumulative = cv2.imread(img_cumulative_path)
                            else:
                                img_cumulative = img.copy()
                            # Add bottom border to image
                            bottom_border = 60
                            BLACK = [0, 0, 0]
                            img = cv2.copyMakeBorder(img, 0, bottom_border, 0, 0, cv2.BORDER_CONSTANT, value=BLACK)

                    # look for a class_name match
                    if obj["class_name"] == class_name:
                        pred_class_name = obj['class_name']
                        if args.gt_box_mode == 'horizontal':
                            bbgt = [float(x) for x in obj["bbox"].split()]
                            gtcx = bbgt[0]
                            gtcy = bbgt[1]
                            gtwidth = bbgt[2]
                            gtheight = bbgt[3]
                            gt_theta = bbgt[4]

                            gtxmin = gtcx
                            gtymin = gtcy
                            gtxmax = gtxmin + np.maximum(0., gtwidth - 1.)
                            gtymax = gtymin + np.maximum(0., gtheight - 1.)

                            bbgt_box = np.array([gtxmin, gtymin, gtxmax, gtymax])
                            horizon_bbox_points = rb_cvt.bbox_to_points(bbgt_box)
                            rotated_bbox_points = rb_cvt.rotate_horizon_bbox_with_theta(horizon_bbox_points, gt_theta)
                            gt_points = rb_cvt.points_to_bbox(rotated_bbox_points)

                            p1 = rotated_bbox_points[0][:-1].astype('int').tolist()
                            p2 = rotated_bbox_points[1][:-1].astype('int').tolist()
                            p3 = rotated_bbox_points[2][:-1].astype('int').tolist()
                            p4 = rotated_bbox_points[3][:-1].astype('int').tolist()
                            gt_plot_points = np.array([p1, p2, p3, p4])
                        else:
                            bbgt = [float(x) for x in obj["bbox"].split()]
                            gtcx = bbgt[0]
                            gtcy = bbgt[1]
                            gtwidth = bbgt[2]
                            gtheight = bbgt[3]
                            gt_theta = bbgt[4]

                            gtxmin = gtcx - (gtwidth / 2)
                            gtymin = gtcy - (gtheight / 2)
                            gtxmax = gtwidth + gtxmin
                            gtymax = gtheight + gtymin

                            bbgt_box = np.array([gtxmin, gtymin, gtxmax, gtymax])

                            horizon_bbox_points = rb_cvt.bbox_to_points(bbgt_box)
                            rotated_bbox_points = rb_cvt.rotate_horizon_bbox_with_theta(horizon_bbox_points, gt_theta)
                            gt_points = rb_cvt.points_to_bbox(rotated_bbox_points)

                            p1 = rotated_bbox_points[0][:-1].astype('int').tolist()
                            p2 = rotated_bbox_points[1][:-1].astype('int').tolist()
                            p3 = rotated_bbox_points[2][:-1].astype('int').tolist()
                            p4 = rotated_bbox_points[3][:-1].astype('int').tolist()
                            gt_plot_points = np.array([p1, p2, p3, p4])

                        if args.pred_box_mode == 'horizontal':
                            cx = bb[0]
                            cy = bb[1]
                            width = bb[2]
                            height = bb[3]
                            theta = -math.radians(bb[4])

                            xmin = cx
                            ymin = cy
                            xmax = xmin + np.maximum(0., width - 1.)
                            ymax = ymin + np.maximum(0., height - 1.)
                            bb_box = np.array([xmin, ymin, xmax, ymax])

                            horizon_bbox_points = rb_cvt.bbox_to_points(bb_box)
                            rotated_bbox_points = rb_cvt.rotate_horizon_bbox_with_theta(horizon_bbox_points, theta)
                            pred_points = rb_cvt.points_to_bbox(rotated_bbox_points)

                            p1 = rotated_bbox_points[0][:-1].astype('int').tolist()
                            p2 = rotated_bbox_points[1][:-1].astype('int').tolist()
                            p3 = rotated_bbox_points[2][:-1].astype('int').tolist()
                            p4 = rotated_bbox_points[3][:-1].astype('int').tolist()
                            pred_plot_points = np.array([p1, p2, p3, p4])

                        else:
                            cx = bb[0]
                            cy = bb[1]
                            width = bb[2]
                            height = bb[3]
                            theta = -math.radians(bb[4])

                            xmin = cx - (width / 2)
                            ymin = cy - (height / 2)
                            xmax = width + xmin
                            ymax = height + ymin
                            bb_box = np.array([xmin, ymin, xmax, ymax])

                            horizon_bbox_points = rb_cvt.bbox_to_points(bb_box)
                            rotated_bbox_points = rb_cvt.rotate_horizon_bbox_with_theta(horizon_bbox_points, theta)
                            pred_points = rb_cvt.points_to_bbox(rotated_bbox_points)

                            p1 = rotated_bbox_points[0][:-1].astype('int').tolist()
                            p2 = rotated_bbox_points[1][:-1].astype('int').tolist()
                            p3 = rotated_bbox_points[2][:-1].astype('int').tolist()
                            p4 = rotated_bbox_points[3][:-1].astype('int').tolist()
                            pred_plot_points = np.array([p1, p2, p3, p4])

                        iou_pred_p1 = pred_points[0] + (width / 2)
                        iou_pred_p2 = pred_points[1] + (height / 2)
                        iou_pred_p3 = pred_points[2] - pred_points[0]
                        iou_pred_p4 = pred_points[3] - pred_points[1]

                        iou_pred_box = np.array([iou_pred_p1, iou_pred_p2, iou_pred_p3, iou_pred_p4])

                        iou_gt_p1 = gt_points[0] + (gtwidth / 2)
                        iou_gt_p2 = gt_points[1] + (gtheight / 2)
                        iou_gt_p3 = gt_points[2] - gt_points[0]
                        iou_gt_p4 = gt_points[3] - gt_points[1]

                        iou_gt_box = np.array([iou_gt_p1, iou_gt_p2, iou_gt_p3, iou_gt_p4])

                        box = np.array([np.append(iou_pred_box, [0], axis=0)], np.float32)
                        gtbox = np.array([np.append(iou_gt_box, [0], axis=0)], np.float32)

                        if args.pred_box_mode == args.gt_box_mode:
                            if args.gpu == "gpu":
                                ov = iou_rotate_calculate(np.array([bb], np.float32), np.array([bbgt], np.float32),
                                                          use_gpu=True, gpu_id=0)
                            else:
                                ov = iou_rotate_calculate(np.array([bb], np.float32), np.array([bbgt], np.float32),
                                                          use_gpu=False, gpu_id=0)
                            gt_match = obj
                        else:
                            if args.gpu == "gpu":
                                ov = iou_rotate_calculate(box, gtbox, use_gpu=True, gpu_id=0)
                            else:
                                ov = iou_rotate_calculate(box, gtbox, use_gpu=False, gpu_id=0)
                            gt_match = obj

                        if ov > ovmax:
                            ovmax = ov
                            gt_match = obj

                        # assign detection as true positive/don't care/false positive
                        if show_animation:
                            status = "NO MATCH FOUND!"  # status is only used in the animation
                        # set minimum overlap
                        min_overlap = MINOVERLAP
                        if specific_iou_flagged:
                            if class_name in specific_iou_classes:
                                index = specific_iou_classes.index(class_name)
                                min_overlap = float(iou_list[index])

                        pred_cx = (pred_points[0] + pred_points[2]) / 2
                        pred_cy = (pred_points[1] + pred_points[3]) / 2
                        gt_cx = (gt_points[0] + gt_points[2]) / 2
                        gt_cy = (gt_points[1] + gt_points[3]) / 2

                        if ov >= min_overlap:
                            if "difficult" not in gt_match:
                                if abs(pred_cx - gt_cx) < 50 and abs(pred_cy - gt_cy) < 50:
                                    if ov > min_overlap:
                                        iou.append(ov)
                                    # true positive
                                    tp[idx] = 1
                                    gt_match["used"] = True
                                    count_true_positives[class_name] += 1
                                    # update the ".json" file
                                    with open(gt_file, 'w') as f:
                                        f.write(json.dumps(ground_truth_data))
                                    if show_animation:
                                        status = "MATCH!"
                                else:
                                    # false positive (multiple detection)
                                    fp[idx] = 1
                                    if show_animation:
                                        status = "REPEATED MATCH!"
                        else:
                            # false positive
                            fp[idx] = 1
                            if ov > 0:
                                status = "INSUFFICIENT OVERLAP"

                        """
                         Draw image to show animation
                        """

                        if show_animation:
                            height, widht = img.shape[:2]
                            # colors (OpenCV works with BGR)
                            white = (255, 255, 255)
                            light_blue = (255, 200, 100)
                            green = (0, 255, 0)
                            light_red = (30, 30, 255)
                            yellow = (255, 255, 0)

                            # 1st line
                            margin = 10
                            v_pos = int(height - margin - (bottom_border / 2.0))
                            text = "Image: " + ground_truth_img[0] + " "
                            img, line_width = draw_text_in_image(img, text, (margin, v_pos), white, 0)
                            text = "Class [" + str(class_index) + "/" + str(n_classes) + "]: " + class_name + " "
                            img, line_width = draw_text_in_image(img, text, (margin + line_width, v_pos), light_blue,
                                                                 line_width)
                            if ovmax != -1:
                                color = light_red
                                if status == "MATCH!":
                                    text = "IoU: {0:.2f}% ".format(ov * 100) + "< {0:.2f}% ".format(min_overlap * 100)
                                    color = green
                                else:
                                    text = "IoU: {0:.2f}% ".format(ov * 100) + ">= {0:.2f}% ".format(min_overlap * 100)
                                img, _ = draw_text_in_image(img, text, (margin + line_width, v_pos), color, line_width)
                            # 2nd line
                            v_pos += int(bottom_border / 2.0)
                            rank_pos = str(idx + 1)  # rank position (idx starts at 0)
                            text = "Detection #rank: " + rank_pos + " confidence: {0:.2f}% ".format(
                                float(detection["confidence"]) * 100)
                            img, line_width = draw_text_in_image(img, text, (margin, v_pos), white, 0)
                            color = light_red
                            if status == "MATCH!":
                                color = green
                            text = "Result: " + status + " "
                            img, line_width = draw_text_in_image(img, text, (margin + line_width, v_pos), color,
                                                                 line_width)
                            font = cv2.FONT_HERSHEY_SIMPLEX
                            # Visualization Coordinate editting

                            cv2.polylines(img, [gt_plot_points], True, color, 2)
                            cv2.polylines(img_cumulative, [gt_plot_points], True, color, 2)
                            cv2.putText(img, class_name, (gt_plot_points[0][0], gt_plot_points[0][1]), font, 0.6,
                                        light_blue, 1,
                                        cv2.LINE_AA)
                            cv2.putText(img_cumulative, class_name, (gt_plot_points[0][0], gt_plot_points[0][1]), font,
                                        0.6, light_blue, 1,
                                        cv2.LINE_AA)
                            # obj["class_name"]
                            cv2.polylines(img, [pred_plot_points], True, color, 2)
                            cv2.polylines(img_cumulative, [pred_plot_points], True, color, 2)
                            cv2.putText(img, pred_class_name, (pred_plot_points[0][0], pred_plot_points[0][1] - 5),
                                        font, 0.6, white, 1,
                                        cv2.LINE_AA)
                            cv2.putText(img_cumulative, pred_class_name,
                                        (pred_plot_points[0][0], pred_plot_points[0][1] - 5), font, 0.6, white, 1,
                                        cv2.LINE_AA)

                            # show image
                            cv2.imshow("Animation", img)
                            cv2.waitKey(5)  # show for 5ms
                            # save image to results
                            output_img_path = results_files_path + "/images/detections_one_by_one/" + class_name + "_detection" + str(
                                i) + ".jpg"
                            cv2.imwrite(output_img_path, img)
                            i += 1

                            # save the image with all the objects drawn to it
                            cv2.imwrite(img_cumulative_path, img_cumulative)
            miou = 0
            if len(iou) != 0:
                miou = sum(iou) / len(iou)
            # compute precision/recall
            cumsum = 0
            for idx, val in enumerate(fp):
                fp[idx] += cumsum
                cumsum += val

            cumsum = 0
            for idx, val in enumerate(tp):
                tp[idx] += cumsum
                cumsum += val

            rec = tp[:]
            for idx, val in enumerate(tp):
                rec[idx] = float(tp[idx]) / gt_counter_per_class[class_name]

            prec = tp[:]
            for idx, val in enumerate(tp):
                if (fp[idx] + tp[idx]) != 0:
                    prec[idx] = float(tp[idx]) / (fp[idx] + tp[idx])
                else:
                    prec[idx] = 0

            ap, mrec, mprec = voc_ap(rec[:], prec[:])
            sum_AP += ap
            sum_maIoU += miou
            text = "{0:.2f}%".format(
                ap * 100) + " = " + class_name + " AP "  # class_name + " AP = {0:.2f}%".format(ap*100)
            """
             Write to results.txt
            """
            rounded_prec = ['%.2f' % elem for elem in prec]
            rounded_rec = ['%.2f' % elem for elem in rec]
            results_file.write(text + "\n Precision: " + str(rounded_prec) + "\n Recall :" + str(rounded_rec) + "\n\n")
            if not args.quiet:
                print(text)
            ap_dictionary[class_name] = ap
            miou_dictionary[class_name] = miou
            n_images = counter_images_per_class[class_name]
            lamr, mr, fppi = log_average_miss_rate(np.array(rec), np.array(fp), n_images)
            lamr_dictionary[class_name] = lamr

            """
             Draw plot
            """
            if draw_plot:
                plt.plot(rec, prec, '-o')
                # add a new penultimate point to the list (mrec[-2], 0.0)
                # since the last line segment (and respective area) do not affect the AP value
                area_under_curve_x = mrec[:-1] + [mrec[-2]] + [mrec[-1]]
                area_under_curve_y = mprec[:-1] + [0.0] + [mprec[-1]]
                plt.fill_between(area_under_curve_x, 0, area_under_curve_y, alpha=0.2, edgecolor='r')
                # set window title
                fig = plt.gcf()  # gcf - get current figure
                fig.canvas.set_window_title('AP ' + class_name)
                # set plot title
                plt.title('class: ' + text)
                # plt.suptitle('This is a somewhat long figure title', fontsize=16)
                # set axis titles
                plt.xlabel('Recall')
                plt.ylabel('Precision')
                # optional - set axes
                axes = plt.gca()  # gca - get current axes
                axes.set_xlim([0.0, 1.0])
                axes.set_ylim([0.0, 1.05])  # .05 to give some extra space
                # Alternative option -> wait for button to be pressed
                # while not plt.waitforbuttonpress(): pass # wait for key display
                # Alternative option -> normal display
                # save the plot
                fig.savefig(results_files_path + "/classes/" + class_name + ".png")
                plt.cla()  # clear axes for next plot

        if show_animation:
            cv2.destroyAllWindows()

        results_file.write("\n# mAP of all classes\n")
        mAP = sum_AP / n_classes
        maIoU = sum_maIoU / n_classes
        text = "mAP = {0:.2f}%".format(mAP * 100)
        text += " maIoU = {0:.2f}%".format(maIoU * 100)
        results_file.write(text + "\n")
        print(text)

    # remove the temp_files directory
    shutil.rmtree(TEMP_FILES_PATH)

    """
     Count total of rotated-horizontal-detection-results
    """
    # iterate through all the files
    det_counter_per_class = {}
    for txt_file in dr_files_list:
        # get lines to list
        lines_list = file_lines_to_list(txt_file)
        for line in lines_list:
            class_name = line.split()[0]
            # check if class is in the ignore list, if yes skip
            if class_name in args.ignore:
                continue
            # count that object
            if class_name in det_counter_per_class:
                det_counter_per_class[class_name] += 1
            else:
                # if class didn't exist yet
                det_counter_per_class[class_name] = 1
    dr_classes = list(det_counter_per_class.keys())

    """
     Plot the total number of occurences of each class in the ground-truth
    """
    if draw_plot:
        window_title = "ground-truth-info"
        plot_title = "ground-truth\n"
        plot_title += "(" + str(len(ground_truth_files_list)) + " files and " + str(n_classes) + " classes)"
        x_label = "Number of objects per class"
        output_path = results_files_path + "/ground-truth-info.png"
        to_show = False
        plot_color = 'forestgreen'
        draw_plot_func(
            gt_counter_per_class,
            n_classes,
            window_title,
            plot_title,
            x_label,
            output_path,
            to_show,
            plot_color,
            '',
        )

    """
     Write number of ground-truth objects per class to results.txt
    """
    with open(results_files_path + "/results.txt", 'a') as results_file:
        results_file.write("\n# Number of ground-truth objects per class\n")
        for class_name in sorted(gt_counter_per_class):
            results_file.write(class_name + ": " + str(gt_counter_per_class[class_name]) + "\n")

    """
     Finish counting true positives
    """
    for class_name in dr_classes:
        # if class exists in detection-result but not in ground-truth then there are no true positives in that class
        if class_name not in gt_classes:
            count_true_positives[class_name] = 0

    """
     Plot the total number of occurences of each class in the "rotated-horizontal-detection-results" folder
    """
    if draw_plot:
        window_title = "rotated-horizontal-detection-results-info"
        # Plot title
        plot_title = "rotated-horizontal-detection-results\n"
        plot_title += "(" + str(len(dr_files_list)) + " files and "
        count_non_zero_values_in_dictionary = sum(int(x) > 0 for x in list(det_counter_per_class.values()))
        plot_title += str(count_non_zero_values_in_dictionary) + " detected classes)"
        # end Plot title
        x_label = "Number of objects per class"
        output_path = results_files_path + "/rotated-horizontal-detection-results-info.png"
        to_show = False
        plot_color = 'forestgreen'
        true_p_bar = count_true_positives
        draw_plot_func(
            det_counter_per_class,
            len(det_counter_per_class),
            window_title,
            plot_title,
            x_label,
            output_path,
            to_show,
            plot_color,
            true_p_bar
        )

    """
     Write number of detected objects per class to results.txt
    """
    with open(results_files_path + "/results.txt", 'a') as results_file:
        results_file.write("\n# Number of detected objects per class\n")
        for class_name in sorted(dr_classes):
            n_det = det_counter_per_class[class_name]
            text = class_name + ": " + str(n_det)
            text += " (tp:" + str(count_true_positives[class_name]) + ""
            text += ", fp:" + str(n_det - count_true_positives[class_name]) + ")\n"
            results_file.write(text)

    """
     Draw log-average miss rate plot (Show lamr of all classes in decreasing order)
    """
    if draw_plot:
        window_title = "lamr"
        plot_title = "log-average miss rate"
        x_label = "log-average miss rate"
        output_path = results_files_path + "/lamr.png"
        to_show = False
        plot_color = 'royalblue'
        draw_plot_func(
            lamr_dictionary,
            n_classes,
            window_title,
            plot_title,
            x_label,
            output_path,
            to_show,
            plot_color,
            ""
        )
    miou_dictionary = dict(sorted(miou_dictionary.items()))
    """
     Draw mAP plot (Show AP's of all classes in decreasing order)
    """
    if draw_plot:
        window_title = "mAP"
        plot_title = "mAP = {0:.2f}%".format(mAP * 100)
        x_label = "Average Precision"
        output_path = results_files_path + "/mAP.png"
        to_show = True
        plot_color = 'royalblue'
        draw_plot_func(
            ap_dictionary,
            n_classes,
            window_title,
            plot_title,
            x_label,
            output_path,
            to_show,
            plot_color,
            ""
        )
    if draw_plot:
        window_title = "maIoU"
        plot_title = "maIoU = {0:.2f}%".format(maIoU * 100)
        x_label = "mIoU"
        output_path = results_files_path + "/maIoU.png"
        to_show = True
        plot_color = 'royalblue'
        draw_plot_func(
            miou_dictionary,
            n_classes,
            window_title,
            plot_title,
            x_label,
            output_path,
            to_show,
            plot_color,
            ""
        )


if __name__ == '__main__':
    # GT_PATH = os.path.join(os.getcwd(), 'input', 'horizontal-ground-truth')
    GT_PATH = os.path.join(os.getcwd(), 'input', 'rotated-ground-truth')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'horizontal-detection-results(faster_rcnn_R_50_FPN_1x)')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'horizontal-detection-results(faster_rcnn_R_50_DC5_3x)')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'horizontal-detection-results(faster_rcnn_R_50_FPN_3x)')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'horizontal-detection-results(faster_rcnn_X_101_FPN_3x)')
    DR_PATH = os.path.join(os.getcwd(), 'input',"horizontal_iter4000")
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'rotated-detection-results(faster_rcnn_R_50_FPN_1x)')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'rotated-detection-results(faster_rcnn_R_50_DC5_3x)')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'rotated-detection-results(faster_rcnn_R_50_FPN_3x)')
    # DR_PATH = os.path.join(os.getcwd(), 'input', 'rotated-detection-results(faster_rcnn_X_101_FPN_3x)')
    IMG_PATH = os.path.join(os.getcwd(), 'input', 'images-optional')
    main(GT_PATH, DR_PATH, IMG_PATH)