fix: add range operator support

fix: add validations for marking scheme and omr response

samples/community/external/UPSC mock/csv/evaluation.json

@@ -1,13 +1,13 @@
 {
   "source_type": "custom",
   "options": {
-    "questions_in_order": ["q1", "q2", "q3", "q4", "q5"],
+    "questions_in_order": ["q1..5"],
     "answers_in_order": ["C", "B", "A", "D", "E"]
   },
   "marking_scheme": {
     "DEFAULT": {"correct": "2", "incorrect": "-1/3", "unmarked": "0" },
     "NO_NEGATIVES": {
-      "questions": ["q1", "q2", "q3"],
+      "questions": ["q1..3"],
       "marking": { "correct": "2", "incorrect": "0", "unmarked": "0" }
     },
     "BONUS_ATTEMPTED": {

samples/community/external/UPSC mock/custom/evaluation.json

@@ -1,14 +1,13 @@
 {
   "source_type": "custom",
   "options": {
-    "questions_in_order": ["q1","q2","q3","q4"],
-    "answers_in_order": ["C", "B", "A", "D"],
-    "should_explain_scoring": true
+    "questions_in_order": ["q1..5"],
+    "answers_in_order": ["C", "B", "A", "D", "E"]
   },
   "marking_scheme": {
     "DEFAULT": {"correct": "2", "incorrect": "-1/3", "unmarked": "0" },
     "NO_NEGATIVES": {
-      "questions": ["q1..3", "q6..100"],
+      "questions": ["q1..3"],
       "marking": { "correct": "2", "incorrect": "0", "unmarked": "0" }
     },
     "BONUS_ATTEMPTED": {
@@ -20,4 +19,4 @@
       "marking": { "unmarked": "2", "correct": "2", "incorrect": "2" }
     }
   }
-}
+}

src/entry.py

@@ -245,7 +245,7 @@ def process_files(
         score = 0
         if evaluation_config is not None:
             score = evaluate_concatenated_response(omr_response, evaluation_config)
-            logger.info(f"Final score: {round(score, 2)} ")
+            logger.info(f"(/{files_counter}) Final score: {round(score, 2)} ")
 
         if tuning_config.outputs.show_image_level >= 2:
             InteractionUtils.show(

src/evaluation.py

@@ -1,11 +1,17 @@
 import os
+import re
 from fractions import Fraction
 
 import numpy as np
 import pandas as pd
 from rich.table import Table
 
 from src.logger import console, logger
+from src.schemas.evaluation_schema import (
+    BONUS_SECTION_PREFIX,
+    DEFAULT_SECTION_KEY,
+    MARKING_VERDICT_TYPES,
+)
 from src.utils.parsing import open_evaluation_with_validation
 
 
@@ -18,9 +24,6 @@ def parse_float_or_fraction(result):
 
 
 class SectionMarkingScheme:
-    DEFAULT_MARKING_SCHEME = "DEFAULT"
-    MARKING_VERDICT_TYPES = ["correct", "incorrect", "unmarked"]
-
     def __init__(self, section_key, section_scheme, empty_val):
         # todo: get local empty_val from qblock
         self.empty_val = empty_val
@@ -32,23 +35,27 @@ def __init__(self, section_key, section_scheme, empty_val):
         }
 
         # DEFAULT marking scheme follows a shorthand
-        if section_key == self.DEFAULT_MARKING_SCHEME:
+        if section_key == DEFAULT_SECTION_KEY:
             self.questions = None
             self.marking = self.parse_scheme_marking(section_scheme)
         else:
-            self.questions = self.parse_scheme_questions(section_scheme["questions"])
+            self.questions = self.parse_questions(section_scheme["questions"])
             self.marking = self.parse_scheme_marking(section_scheme["marking"])
 
     def parse_scheme_marking(self, marking):
         parsed_marking = {}
-        for verdict_type in self.MARKING_VERDICT_TYPES:
+        for verdict_type in MARKING_VERDICT_TYPES:
             result = marking[verdict_type]
             if type(result) == str:
                 result = parse_float_or_fraction(result)
-
-                if result > 0 and verdict_type == "incorrect":
+                section_key = self.section_key
+                if (
+                    result > 0
+                    and verdict_type == "incorrect"
+                    and not section_key.startswith(BONUS_SECTION_PREFIX)
+                ):
                     logger.warning(
-                        f"Found positive marks({round(result, 2)}) for incorrect answer in the schema '{self.section_key}'. Is this a Bonus Question?"
+                        f"Found positive marks({round(result, 2)}) for incorrect answer in the schema '{section_key}'. For Bonus sections, add a prefix 'BONUS_' to them."
                     )
             elif type(result) == list:
                 result = map(parse_float_or_fraction, result)
@@ -57,12 +64,33 @@ def parse_scheme_marking(self, marking):
 
         return parsed_marking
 
-    def parse_scheme_questions(self, questions):
-        # TODO: simplify this in schema itself
-        # if(questions === "all"): <- handle this case in top parsing + validation
-
-        # TODO: parse the range operator regex here
-        parsed_questions = questions
+    @staticmethod
+    def parse_question_string(question_string):
+        if "." in question_string:
+            question_prefix, start, end = re.findall(
+                r"([^\.\d]+)(\d+)\.\.(\d+)", question_string
+            )[0]
+            start, end = int(start), int(end)
+            if start >= end:
+                raise Exception(
+                    f"Invalid range in question string: {question_string}, start: {start} is not less than end: {end}"
+                )
+            return [
+                f"{question_prefix}{question_number}"
+                for question_number in range(start, end + 1)
+            ]
+        else:
+            return [question_string]
+
+    @staticmethod
+    def parse_questions(questions):
+        parsed_questions = []
+        for question_string in questions:
+            questions_array = SectionMarkingScheme.parse_question_string(
+                question_string
+            )
+            # todo: make a set and validate uniqueness, exhaustiveness etc
+            parsed_questions.extend(questions_array)
         return parsed_questions
 
     def get_question_verdict(self, marked_answer, correct_answer):
@@ -76,7 +104,7 @@ def get_question_verdict(self, marked_answer, correct_answer):
 
     def update_streaks_for_verdict(self, question_verdict):
         current_streak = self.streaks[question_verdict]
-        for verdict_type in self.MARKING_VERDICT_TYPES:
+        for verdict_type in MARKING_VERDICT_TYPES:
             if question_verdict == verdict_type:
                 # increase current streak
                 self.streaks[verdict_type] = current_streak + 1
@@ -99,13 +127,12 @@ def match_answer(self, marked_answer, correct_answer):
 
 class EvaluationConfig:
     def __init__(self, local_evaluation_path, template, curr_dir):
-        evaluation_json = open_evaluation_with_validation(
-            local_evaluation_path, template, curr_dir
-        )
+        evaluation_json = open_evaluation_with_validation(local_evaluation_path)
         options = evaluation_json["options"]
         self.should_explain_scoring = options.get("should_explain_scoring", False)
         if self.should_explain_scoring:
             self.prepare_explanation_table()
+
         marking_scheme = evaluation_json["marking_scheme"]
         if evaluation_json["source_type"] == "csv":
             csv_path = curr_dir.joinpath(options["answer_key_path"])
@@ -119,21 +146,73 @@ def __init__(self, local_evaluation_path, template, curr_dir):
             # TODO: later parse complex answer schemes from csv itself (ans strings)
             # TODO: validate each row to contain a (qNo, <ans string/>) pair
         else:
-            # - if source_type = custom, template should have all valid qNos (in instance?)
             self.questions_in_order = self.parse_questions_in_order(
                 options["questions_in_order"]
             )
             self.answers_in_order = options["answers_in_order"]
+
         self.marking_scheme = {}
         for (section_key, section_scheme) in marking_scheme.items():
             # instance will allow easy readability, extensibility as well as streak sample
             self.marking_scheme[section_key] = SectionMarkingScheme(
                 section_key, section_scheme, template.global_empty_val
             )
 
+    def validate_all(self, omr_response):
+        questions_in_order, answers_in_order = (
+            self.questions_in_order,
+            self.answers_in_order,
+        )
+        if len(questions_in_order) != len(answers_in_order):
+            logger.critical(
+                f"questions_in_order: {questions_in_order}\nanswers_in_order: {answers_in_order}"
+            )
+            raise Exception(
+                f"Unequal lengths for questions_in_order and answers_in_order"
+            )
+
+        section_questions = set()
+        for (section_key, section_scheme) in self.marking_scheme.items():
+            if section_key == DEFAULT_SECTION_KEY:
+                continue
+            current_set = set(section_scheme.questions)
+            if not section_questions.isdisjoint(current_set):
+                raise Exception(
+                    f"Section '{section_key}' has overlapping question(s) with other sections"
+                )
+            section_questions = section_questions.union(current_set)
+
+        answer_key_questions = set(questions_in_order)
+        if answer_key_questions != section_questions:
+            if answer_key_questions.issuperset(section_questions):
+                missing_questions = sorted(
+                    answer_key_questions.difference(section_questions)
+                )
+                logger.critical(f"Missing marking scheme for: {missing_questions}")
+                raise Exception(
+                    f"Some questions from the answer key are missing from the marking scheme"
+                )
+            else:
+                missing_questions = sorted(
+                    section_questions.difference(answer_key_questions)
+                )
+                logger.critical(f"Missing answer key for: {missing_questions}")
+                raise Exception(
+                    f"Some questions from the marking scheme are missing from the answer key"
+                )
+
+        omr_response_questions = set(omr_response.keys())
+        if answer_key_questions.issuperset(omr_response_questions):
+            missing_questions = sorted(
+                answer_key_questions.difference(omr_response_questions)
+            )
+            logger.critical(f"Missing OMR response for: {missing_questions}")
+            raise Exception(
+                f"Some questions from the answer key are missing for the given OMR response"
+            )
+
     def parse_questions_in_order(self, questions_in_order):
-        # TODO: parse range operators here as well
-        return questions_in_order
+        return SectionMarkingScheme.parse_questions(questions_in_order)
 
     def prepare_explanation_table(self):
         table = Table(show_lines=True)
@@ -176,23 +255,21 @@ def conditionally_print_explanation(self):
 
 
 def evaluate_concatenated_response(concatenated_response, evaluation_config):
-    # first go with answers_in_order object, later just export it as csv to get csv format for docs.
+    evaluation_config.validate_all(concatenated_response)
     questions_in_order, answers_in_order, marking_scheme = map(
         evaluation_config.__dict__.get,
         ["questions_in_order", "answers_in_order", "marking_scheme"],
     )
     QUESTION_WISE_SCHEMES = {}
     for (section_key, section_marking_scheme) in marking_scheme.items():
-        if section_key == SectionMarkingScheme.DEFAULT_MARKING_SCHEME:
+        if section_key == DEFAULT_SECTION_KEY:
             default_marking_scheme = marking_scheme[section_key]
         else:
             for q in section_marking_scheme.questions:
                 QUESTION_WISE_SCHEMES[q] = section_marking_scheme
-
     score = 0.0
     for q_index, question in enumerate(questions_in_order):
         correct_answer = answers_in_order[q_index]
-        # TODO: add validation for existence of each question in response keys
         marked_answer = concatenated_response[question]
         question_marking_scheme = QUESTION_WISE_SCHEMES.get(
             question, default_marking_scheme

src/schemas/evaluation_schema.py

@@ -1,3 +1,6 @@
+DEFAULT_SECTION_KEY = "DEFAULT"
+BONUS_SECTION_PREFIX = "BONUS"
+MARKING_VERDICT_TYPES = ["correct", "incorrect", "unmarked"]
 marking_object_properties = {
     "additionalProperties": False,
     "required": ["correct", "incorrect", "unmarked"],
@@ -40,8 +43,9 @@
         "options": {"type": "object"},
         "marking_scheme": {
             "type": "object",
+            "required": [DEFAULT_SECTION_KEY],
             "patternProperties": {
-                "DEFAULT": marking_object_properties,
+                DEFAULT_SECTION_KEY: marking_object_properties,
                 "^(?!DEFAULT$).*": {
                     "additionalProperties": False,
                     "required": ["marking", "questions"],

src/utils/parsing.py

@@ -81,11 +81,9 @@ def open_template_with_defaults(template_path):
         exit()
 
 
-def open_evaluation_with_validation(evaluation_path, template, curr_dir):
+def open_evaluation_with_validation(evaluation_path):
     user_evaluation_config = load_json(evaluation_path)
-    is_valid = validate_evaluation_json(
-        user_evaluation_config, evaluation_path, template, curr_dir
-    )
+    is_valid = validate_evaluation_json(user_evaluation_config, evaluation_path)
 
     if is_valid:
         return user_evaluation_config

src/utils/validations.py

@@ -24,7 +24,7 @@ def parse_validation_error(error):
     )
 
 
-def validate_evaluation_json(json_data, evaluation_path, template, curr_dir):
+def validate_evaluation_json(json_data, evaluation_path):
     logger.info("Validating evaluation.json...")
     try:
         validate(instance=json_data, schema=SCHEMA_JSONS["evaluation"])
@@ -51,12 +51,6 @@ def validate_evaluation_json(json_data, evaluation_path, template, curr_dir):
         logger.critical(f"Provided Evaluation JSON is Invalid: '{evaluation_path}'")
         return False
 
-    # TODO: also validate these
-    # - All mentioned qNos in sections should be present in template.json
-    # - All ranges in questions_order should be exhaustive too
-    # - All keys of sections should be present in keys of marking
-    # - Sections should be mutually exclusive
-
     logger.info("Evaluation JSON validated successfully")
     return True