feat: support for multiple-correct-weighted answer key (Udayraj123#166)

* feat: support for multiple-correct-weighted answer key; minor refactor evaluation schema

* fix: update validation

.pre-commit-config.yaml

@@ -41,7 +41,7 @@ repos:
     hooks:
       - id: pytest-on-commit
         name: Running single sample test
-        entry:  pytest -rfpsxEX --disable-warnings --verbose -k sample1
+        entry:  python3 -m pytest -rfpsxEX --disable-warnings --verbose -k sample1
         language: system
         pass_filenames: false
         always_run: true
@@ -51,7 +51,7 @@ repos:
     hooks:
       - id: pytest-on-push
         name: Running all tests before push...
-        entry: pytest -rfpsxEX --disable-warnings --verbose --durations=3
+        entry: python3 -m pytest -rfpsxEX --disable-warnings --verbose --durations=3
         language: system
         pass_filenames: false
         always_run: true

samples/sample2/answer_key.csv → samples/answer-key/using-csv/answer_key.csv

@@ -1,5 +1,5 @@
-q1,B
+q1,C
 q2,E
 q3,A
-q4,C
+q4,B
 q5,B

samples/sample2/evaluation.json → samples/answer-key/using-csv/evaluation.json

@@ -4,7 +4,7 @@
     "answer_key_csv_path": "answer_key.csv",
     "should_explain_scoring": true
   },
-  "marking_scheme": {
+  "marking_schemes": {
     "DEFAULT": {
       "correct": "1",
       "incorrect": "0",

samples/answer-key/using-csv/template.json

@@ -0,0 +1,35 @@
+{
+  "pageDimensions": [
+    300,
+    400
+  ],
+  "bubbleDimensions": [
+    25,
+    25
+  ],
+  "preProcessors": [
+    {
+      "name": "CropPage",
+      "options": {
+        "morphKernel": [
+          10,
+          10
+        ]
+      }
+    }
+  ],
+  "fieldBlocks": {
+    "MCQ_Block_1": {
+      "fieldType": "QTYPE_MCQ5",
+      "origin": [
+        65,
+        60
+      ],
+      "fieldLabels": [
+        "q1..5"
+      ],
+      "labelsGap": 52,
+      "bubblesGap": 41
+    }
+  }
+}

samples/answer-key/weighted-answers/evaluation.json

@@ -0,0 +1,35 @@
+{
+  "source_type": "custom",
+  "options": {
+    "questions_in_order": [
+      "q1..5"
+    ],
+    "answers_in_order": [
+      "C",
+      "E",
+      [
+        "A",
+        "C"
+      ],
+      [
+        [
+          "B",
+          2
+        ],
+        [
+          "C",
+          "3/2"
+        ]
+      ],
+      "C"
+    ],
+    "should_explain_scoring": true
+  },
+  "marking_schemes": {
+    "DEFAULT": {
+      "correct": "3",
+      "incorrect": "-1",
+      "unmarked": "0"
+    }
+  }
+}

samples/answer-key/weighted-answers/template.json

@@ -0,0 +1,35 @@
+{
+  "pageDimensions": [
+    300,
+    400
+  ],
+  "bubbleDimensions": [
+    25,
+    25
+  ],
+  "preProcessors": [
+    {
+      "name": "CropPage",
+      "options": {
+        "morphKernel": [
+          10,
+          10
+        ]
+      }
+    }
+  ],
+  "fieldBlocks": {
+    "MCQ_Block_1": {
+      "fieldType": "QTYPE_MCQ5",
+      "origin": [
+        65,
+        60
+      ],
+      "fieldLabels": [
+        "q1..5"
+      ],
+      "labelsGap": 52,
+      "bubblesGap": 41
+    }
+  }
+}

samples/community/UPSC-mock/evaluation.json

@@ -8,7 +8,7 @@
     ],
     "should_explain_scoring": true
   },
-  "marking_scheme": {
+  "marking_schemes": {
     "DEFAULT": {
       "correct": "2",
       "incorrect": "-2/3",

samples/community/UmarFarootAPS/answer_key.csv

@@ -2,8 +2,8 @@ q1,C
 q2,C
 q3,"D,E"
 q4,"A,AB"
-q5,"['A', ['1', '-2/3', '0']]"
-q6,"['A', ['2']]"
+q5,"[['A', '1'], ['B', '2']]"
+q6,"['A', 'B']"
 q7,C
 q8,D
 q9,B

samples/community/UmarFarootAPS/evaluation.json

@@ -4,7 +4,7 @@
     "answer_key_csv_path": "answer_key.csv",
     "should_explain_scoring": true
   },
-  "marking_scheme": {
+  "marking_schemes": {
     "DEFAULT": {
       "correct": "1",
       "incorrect": "0",

samples/sample4/evaluation.json

@@ -11,15 +11,19 @@
       "B",
       "D",
       "C",
-      "BC",
+      [
+        "B",
+        "C",
+        "BC"
+      ],
       "A",
       "C",
       "D",
       "C"
     ],
     "should_explain_scoring": true
   },
-  "marking_scheme": {
+  "marking_schemes": {
     "DEFAULT": {
       "correct": "3",
       "incorrect": "-1",

samples/sample5/evaluation.json

@@ -33,7 +33,7 @@
     ],
     "should_explain_scoring": true
   },
-  "marking_scheme": {
+  "marking_schemes": {
     "DEFAULT": {
       "correct": "1",
       "incorrect": "0",

src/evaluation.py

@@ -22,123 +22,127 @@
 
 
 class AnswerMatcher:
-    def __init__(self, answer_item, marking_scheme):
-        self.answer_type = self.get_answer_type(answer_item)
-        self.parsed_answer = self.parse_answer_item(answer_item)
-        self.set_defaults_from_scheme(marking_scheme)
-        self.marking_scheme = marking_scheme
-
-    def get_answer_type(self, answer_item):
-        item_type = type(answer_item)
-        if item_type == str:
+    def __init__(self, answer_item, section_marking_scheme):
+        self.section_marking_scheme = section_marking_scheme
+        self.answer_item = answer_item
+        self.answer_type = self.validate_and_get_answer_type(answer_item)
+        self.set_defaults_from_scheme(section_marking_scheme)
+
+    @staticmethod
+    def is_a_marking_score(answer_element):
+        # Note: strict type checking is already done at schema validation level,
+        # Here we focus on overall struct type
+        return type(answer_element) == str or type(answer_element) == int
+
+    @staticmethod
+    def is_standard_answer(answer_element):
+        return type(answer_element) == str and len(answer_element) >= 1
+
+    def validate_and_get_answer_type(self, answer_item):
+        if self.is_standard_answer(answer_item):
             return "standard"
-        elif item_type == list:
+        elif type(answer_item) == list:
             if (
+                # Array of answer elements: ['A', 'B', 'AB']
                 len(answer_item) >= 2
-                and type(answer_item[0]) == str
-                and type(answer_item[1]) == str
+                and all(
+                    self.is_standard_answer(answers_or_score)
+                    for answers_or_score in answer_item
+                )
             ):
                 return "multiple-correct"
             elif (
-                len(answer_item) == 2
-                and type(answer_item[0]) == str
-                and type(answer_item[1]) == list
+                # Array of two-tuples: [['A', 1], ['B', 1], ['C', 3], ['AB', 2]]
+                len(answer_item) >= 1
+                and all(
+                    type(answer_and_score) == list and len(answer_and_score) == 2
+                    for answer_and_score in answer_item
+                )
+                and all(
+                    self.is_standard_answer(allowed_answer)
+                    and self.is_a_marking_score(answer_score)
+                    for allowed_answer, answer_score in answer_item
+                )
             ):
                 return "multiple-correct-weighted"
-            elif (
-                len(answer_item) == 2
-                and type(answer_item[0]) in list
-                and type(answer_item[1]) == list
-            ):
-                return "answer-weights"
-            else:
-                logger.critical(
-                    f"Unable to determine answer type for answer item: {answer_item}"
-                )
-                raise Exception("Unable to determine answer type")
 
-    def parse_answer_item(self, answer_item):
-        return answer_item
+        logger.critical(
+            f"Unable to determine answer type for answer item: {answer_item}"
+        )
+        raise Exception("Unable to determine answer type")
 
-    def set_defaults_from_scheme(self, marking_scheme):
+    def set_defaults_from_scheme(self, section_marking_scheme):
         answer_type = self.answer_type
-        self.empty_val = marking_scheme.empty_val
-        parsed_answer = self.parsed_answer
-        self.marking = deepcopy(marking_scheme.marking)
+        self.empty_val = section_marking_scheme.empty_val
+        answer_item = self.answer_item
+        self.marking = deepcopy(section_marking_scheme.marking)
         # TODO: reuse part of parse_scheme_marking here -
         if answer_type == "standard":
             # no local overrides
             pass
         elif answer_type == "multiple-correct":
-            # no local overrides
-            for allowed_answer in parsed_answer:
+            # override marking scheme scores for each allowed answer
+            for allowed_answer in answer_item:
                 self.marking[f"correct-{allowed_answer}"] = self.marking["correct"]
         elif answer_type == "multiple-correct-weighted":
-            custom_marking = list(map(parse_float_or_fraction, parsed_answer[1]))
-            verdict_types_length = min(len(MARKING_VERDICT_TYPES), len(custom_marking))
-            # override the given marking
-            for i in range(verdict_types_length):
-                verdict_type = MARKING_VERDICT_TYPES[i]
-                self.marking[verdict_type] = custom_marking[i]
-
-            if type(parsed_answer[0] == str):
-                allowed_answer = parsed_answer[0]
-                self.marking[f"correct-{allowed_answer}"] = self.marking["correct"]
-            else:
-                for allowed_answer in parsed_answer[0]:
-                    self.marking[f"correct-{allowed_answer}"] = self.marking["correct"]
+            # Note: No override using marking scheme as answer scores are provided in answer_item
+            for allowed_answer, answer_score in answer_item:
+                self.marking[f"correct-{allowed_answer}"] = parse_float_or_fraction(
+                    answer_score
+                )
 
     def get_marking_scheme(self):
-        return self.marking_scheme
+        return self.section_marking_scheme
 
     def get_section_explanation(self):
         answer_type = self.answer_type
         if answer_type in ["standard", "multiple-correct"]:
-            return self.marking_scheme.section_key
+            return self.section_marking_scheme.section_key
         elif answer_type == "multiple-correct-weighted":
             return f"Custom: {self.marking}"
 
     def get_verdict_marking(self, marked_answer):
         answer_type = self.answer_type
+        question_verdict = "incorrect"
         if answer_type == "standard":
             question_verdict = self.get_standard_verdict(marked_answer)
-            return question_verdict, self.marking[question_verdict]
         elif answer_type == "multiple-correct":
             question_verdict = self.get_multiple_correct_verdict(marked_answer)
-            return question_verdict, self.marking[question_verdict]
         elif answer_type == "multiple-correct-weighted":
-            question_verdict = self.get_multi_weighted_verdict(marked_answer)
-            return question_verdict, self.marking[question_verdict]
+            question_verdict = self.get_multiple_correct_weighted_verdict(marked_answer)
+        return question_verdict, self.marking[question_verdict]
 
     def get_standard_verdict(self, marked_answer):
-        parsed_answer = self.parsed_answer
+        allowed_answer = self.answer_item
         if marked_answer == self.empty_val:
             return "unmarked"
-        elif marked_answer == parsed_answer:
+        elif marked_answer == allowed_answer:
             return "correct"
         else:
             return "incorrect"
 
-    def get_multi_weighted_verdict(self, marked_answer):
-        return self.get_multiple_correct_verdict(marked_answer)
-
     def get_multiple_correct_verdict(self, marked_answer):
-        parsed_answer = self.parsed_answer
+        allowed_answers = self.answer_item
         if marked_answer == self.empty_val:
             return "unmarked"
-        elif marked_answer in parsed_answer:
+        elif marked_answer in allowed_answers:
+            return f"correct-{marked_answer}"
+        else:
+            return "incorrect"
+
+    def get_multiple_correct_weighted_verdict(self, marked_answer):
+        allowed_answers = [
+            allowed_answer for allowed_answer, _answer_score in self.answer_item
+        ]
+        if marked_answer == self.empty_val:
+            return "unmarked"
+        elif marked_answer in allowed_answers:
             return f"correct-{marked_answer}"
         else:
             return "incorrect"
 
     def __str__(self):
-        answer_type, parsed_answer = self.answer_type, self.parsed_answer
-        if answer_type == "multiple-correct":
-            return str(parsed_answer)
-        elif answer_type == "multiple-correct-weighted":
-            return f"{parsed_answer[0]}"
-            # TODO: multi-lines in multi-weights
-        return parsed_answer
+        return f"{self.answer_item}"
 
 
 class SectionMarkingScheme:
@@ -154,6 +158,9 @@ def __init__(self, section_key, section_scheme, empty_val):
             self.questions = parse_fields(section_key, section_scheme["questions"])
             self.marking = self.parse_scheme_marking(section_scheme["marking"])
 
+    def __str__(self):
+        return self.section_key
+
     def parse_scheme_marking(self, marking):
         parsed_marking = {}
         for verdict_type in MARKING_VERDICT_TYPES:
@@ -184,15 +191,13 @@ class EvaluationConfig:
     def __init__(self, curr_dir, evaluation_path, template, tuning_config):
         self.path = evaluation_path
         evaluation_json = open_evaluation_with_validation(evaluation_path)
-        options, marking_scheme, source_type = map(
-            evaluation_json.get, ["options", "marking_scheme", "source_type"]
+        options, marking_schemes, source_type = map(
+            evaluation_json.get, ["options", "marking_schemes", "source_type"]
         )
         self.should_explain_scoring = options.get("should_explain_scoring", False)
         self.has_non_default_section = False
         self.exclude_files = []
 
-        marking_scheme = marking_scheme
-
         if source_type == "csv":
             csv_path = curr_dir.joinpath(options["answer_key_csv_path"])
             if not os.path.exists(csv_path):
@@ -286,21 +291,21 @@ def __init__(self, curr_dir, evaluation_path, template, tuning_config):
 
         self.validate_questions(answers_in_order)
 
-        self.marking_scheme, self.question_to_scheme = {}, {}
-        for section_key, section_scheme in marking_scheme.items():
+        self.section_marking_schemes, self.question_to_scheme = {}, {}
+        for section_key, section_scheme in marking_schemes.items():
             section_marking_scheme = SectionMarkingScheme(
                 section_key, section_scheme, template.global_empty_val
             )
             if section_key != DEFAULT_SECTION_KEY:
-                self.marking_scheme[section_key] = section_marking_scheme
+                self.section_marking_schemes[section_key] = section_marking_scheme
                 for q in section_marking_scheme.questions:
                     # TODO: check the answer key for custom scheme here?
                     self.question_to_scheme[q] = section_marking_scheme
                 self.has_non_default_section = True
             else:
                 self.default_marking_scheme = section_marking_scheme
 
-        self.validate_marking_scheme()
+        self.validate_marking_schemes()
 
         self.question_to_answer_matcher = self.parse_answers_and_map_questions(
             answers_in_order
@@ -360,10 +365,13 @@ def get_exclude_files(self):
     @staticmethod
     def parse_answer_column(answer_column):
         if answer_column[0] == "[":
+            # multiple-correct-weighted or multiple-correct
             parsed_answer = ast.literal_eval(answer_column)
         elif "," in answer_column:
+            # multiple-correct
             parsed_answer = answer_column.split(",")
         else:
+            # single-correct
             parsed_answer = answer_column
         return parsed_answer
 
@@ -385,12 +393,13 @@ def validate_answers(self, answers_in_order, tuning_config):
                             multi_marked_answer = True
                             break
                 if answer_type == "multiple-correct-weighted":
-                    if len(answer_item[0]) > 1:
-                        multi_marked_answer = True
+                    for single_answer, _answer_score in answer_item:
+                        if len(single_answer) > 1:
+                            multi_marked_answer = True
 
                 if multi_marked_answer:
                     raise Exception(
-                        f"Answer key contains multiple correct answer(s), but filter_out_multimarked_files is True. Scoring will get skipped."
+                        f"Provided answer key contains multiple correct answer(s), but config.filter_out_multimarked_files is True. Scoring will get skipped."
                     )
 
     def validate_questions(self, answers_in_order):
@@ -406,10 +415,10 @@ def validate_questions(self, answers_in_order):
                 f"Unequal lengths for questions_in_order and answers_in_order ({len_questions_in_order} != {len_answers_in_order})"
             )
 
-    def validate_marking_scheme(self):
-        marking_scheme = self.marking_scheme
+    def validate_marking_schemes(self):
+        section_marking_schemes = self.section_marking_schemes
         section_questions = set()
-        for section_key, section_scheme in marking_scheme.items():
+        for section_key, section_scheme in section_marking_schemes.items():
             if section_key == DEFAULT_SECTION_KEY:
                 continue
             current_set = set(section_scheme.questions)
@@ -431,9 +440,15 @@ def parse_answers_and_map_questions(self, answers_in_order):
         question_to_answer_matcher = {}
         for question, answer_item in zip(self.questions_in_order, answers_in_order):
             section_marking_scheme = self.get_marking_scheme_for_question(question)
-            question_to_answer_matcher[question] = AnswerMatcher(
-                answer_item, section_marking_scheme
-            )
+            answer_matcher = AnswerMatcher(answer_item, section_marking_scheme)
+            question_to_answer_matcher[question] = answer_matcher
+            if (
+                answer_matcher.answer_type == "multiple-correct-weighted"
+                and section_marking_scheme.section_key != DEFAULT_SECTION_KEY
+            ):
+                logger.warning(
+                    f"The custom scheme '{section_marking_scheme}' will not apply to question '{question}' as it will use the given answer weights f{answer_item}"
+                )
         return question_to_answer_matcher
 
     # Then unfolding lower abstraction levels
@@ -481,9 +496,11 @@ def conditionally_add_explanation(
                     str.title(question_verdict),
                     str(round(delta, 2)),
                     str(round(next_score, 2)),
-                    answer_matcher.get_section_explanation()
-                    if self.has_non_default_section
-                    else None,
+                    (
+                        answer_matcher.get_section_explanation()
+                        if self.has_non_default_section
+                        else None
+                    ),
                 ]
                 if item is not None
             ]

src/schemas/evaluation_schema.py

@@ -18,6 +18,7 @@
     "required": ["correct", "incorrect", "unmarked"],
     "type": "object",
     "properties": {
+        # TODO: can support streak marking if we allow array of marking_scores here
         "correct": marking_score,
         "incorrect": marking_score,
         "unmarked": marking_score,
@@ -31,12 +32,12 @@
     "description": "OMRChecker evaluation schema i.e. the marking scheme",
     "type": "object",
     "additionalProperties": True,
-    "required": ["source_type", "options", "marking_scheme"],
+    "required": ["source_type", "options", "marking_schemes"],
     "properties": {
         "additionalProperties": False,
         "source_type": {"type": "string", "enum": ["csv", "custom"]},
         "options": {"type": "object"},
-        "marking_scheme": {
+        "marking_schemes": {
             "type": "object",
             "required": [DEFAULT_SECTION_KEY],
             "patternProperties": {
@@ -102,57 +103,38 @@
                                         "type": "array",
                                         "items": {
                                             "oneOf": [
-                                                # "standard": single correct, multimarked single-correct
-                                                # Example: "q1" --> 'AB'
+                                                # "standard": single correct, multi-marked single-correct
+                                                # Example: "q1" --> '67'
                                                 {"type": "string"},
-                                                # "multiple-correct": multiple correct answers (for ambiguos/bonus questions)
+                                                # "multiple-correct": multiple-correct (for ambiguous/bonus questions)
                                                 # Example: "q1" --> [ 'A', 'B' ]
                                                 {
                                                     "type": "array",
                                                     "items": {"type": "string"},
                                                     "minItems": 2,
                                                 },
-                                                # "multiple-correct-weighted": array of answer-wise weights
-                                                # Example: "q1" --> [['A', 1], ['B', 2], ['C', 3]]
+                                                # "multiple-correct-weighted": array of answer-wise weights (marking scheme not applicable)
+                                                # Example 1: "q1" --> [['A', 1], ['B', 2], ['C', 3]] or
+                                                # Example 2: "q2" --> [['A', 1], ['B', 1], ['AB', 2]]
                                                 {
                                                     "type": "array",
-                                                    "items": False,
-                                                    "maxItems": 2,
-                                                    "minItems": 2,
-                                                    "prefixItems": [
-                                                        {"type": "string"},
-                                                        {
-                                                            "type": "array",
-                                                            "items": marking_score,
-                                                            "minItems": 1,
-                                                            "maxItems": 3,
-                                                        },
-                                                    ],
+                                                    "items": {
+                                                        "type": "array",
+                                                        "items": False,
+                                                        "minItems": 2,
+                                                        "maxItems": 2,
+                                                        "prefixItems": [
+                                                            {"type": "string"},
+                                                            marking_score,
+                                                        ],
+                                                    },
                                                 },
+                                                # Multiple-correct with custom marking scheme
+                                                # ["A", ["1", "2", "3"]],
+                                                # [["A", "B", "AB"], ["1", "2", "3"]]
                                             ],
                                         },
                                     },
-                                    {
-                                        # TODO: answer_weight format
-                                        "type": "array",  # two column array for weights
-                                        "items": False,
-                                        "maxItems": 2,
-                                        "minItems": 2,
-                                        "prefixItems": [
-                                            {
-                                                "type": "array",
-                                                "items": {"type": "string"},
-                                                "minItems": 2,
-                                                "maxItems": 2,
-                                            },
-                                            {
-                                                "type": "array",
-                                                "items": marking_score,
-                                                "minItems": 1,
-                                                "maxItems": 3,
-                                            },
-                                        ],
-                                    },
                                 ]
                             },
                             "questions_in_order": ARRAY_OF_STRINGS,

src/tests/__snapshots__/test_all_samples.ambr

@@ -1,4 +1,39 @@
 # serializer version: 1
+# name: test_run_answer_key_using_csv
+  dict({
+    'Manual/ErrorFiles.csv': '''
+      "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
+
+    ''',
+    'Manual/MultiMarkedFiles.csv': '''
+      "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
+
+    ''',
+    'Results/Results_05AM.csv': '''
+      "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
+      "adrian_omr.png","samples/answer-key/using-csv/adrian_omr.png","outputs/answer-key/using-csv/CheckedOMRs/adrian_omr.png","5.0","C","E","A","B","B"
+
+    ''',
+  })
+# ---
+# name: test_run_answer_key_weighted_answers
+  dict({
+    'images/Manual/ErrorFiles.csv': '''
+      "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
+
+    ''',
+    'images/Manual/MultiMarkedFiles.csv': '''
+      "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
+
+    ''',
+    'images/Results/Results_05AM.csv': '''
+      "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
+      "adrian_omr.png","samples/answer-key/weighted-answers/images/adrian_omr.png","outputs/answer-key/weighted-answers/images/CheckedOMRs/adrian_omr.png","5.5","B","E","A","C","B"
+      "adrian_omr_2.png","samples/answer-key/weighted-answers/images/adrian_omr_2.png","outputs/answer-key/weighted-answers/images/CheckedOMRs/adrian_omr_2.png","10.0","C","E","A","B","B"
+
+    ''',
+  })
+# ---
 # name: test_run_community_Antibodyy
   dict({
     'Manual/ErrorFiles.csv': '''
@@ -97,7 +132,7 @@
     'scans/Results/Results_05AM.csv': '''
       "file_id","input_path","output_path","score","Roll_no","q1","q2","q3","q4","q5","q6","q7","q8","q9","q10","q11","q12","q13","q14","q15","q16","q17","q18","q19","q20","q21","q22","q23","q24","q25","q26","q27","q28","q29","q30","q31","q32","q33","q34","q35","q36","q37","q38","q39","q40","q41","q42","q43","q44","q45","q46","q47","q48","q49","q50","q51","q52","q53","q54","q55","q56","q57","q58","q59","q60","q61","q62","q63","q64","q65","q66","q67","q68","q69","q70","q71","q72","q73","q74","q75","q76","q77","q78","q79","q80","q81","q82","q83","q84","q85","q86","q87","q88","q89","q90","q91","q92","q93","q94","q95","q96","q97","q98","q99","q100","q101","q102","q103","q104","q105","q106","q107","q108","q109","q110","q111","q112","q113","q114","q115","q116","q117","q118","q119","q120","q121","q122","q123","q124","q125","q126","q127","q128","q129","q130","q131","q132","q133","q134","q135","q136","q137","q138","q139","q140","q141","q142","q143","q144","q145","q146","q147","q148","q149","q150","q151","q152","q153","q154","q155","q156","q157","q158","q159","q160","q161","q162","q163","q164","q165","q166","q167","q168","q169","q170","q171","q172","q173","q174","q175","q176","q177","q178","q179","q180","q181","q182","q183","q184","q185","q186","q187","q188","q189","q190","q191","q192","q193","q194","q195","q196","q197","q198","q199","q200"
       "scan-type-1.jpg","samples/community/UmarFarootAPS/scans/scan-type-1.jpg","outputs/community/UmarFarootAPS/scans/CheckedOMRs/scan-type-1.jpg","49.0","2468","A","C","B","C","A","D","B","C","B","D","C","A","C","D","B","C","A","B","C","A","C","B","D","C","A","B","D","C","A","C","B","D","B","A","C","D","B","C","A","C","D","A","C","D","A","B","D","C","A","C","D","B","C","A","C","D","B","C","D","A","B","C","B","C","D","B","D","A","C","B","D","A","B","C","B","A","C","D","B","A","C","B","C","B","A","D","B","A","C","D","B","D","B","C","B","D","A","C","B","C","B","C","D","B","C","A","B","C","A","D","C","B","D","B","A","B","C","D","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","B","A","C","B","A","C","A","B","C","B","C","B","A","C","A","C","B","B","C","B","A","C","A","B","A","B","A","B","C","D","B","C","A","C","D","C","A","C","B","A","C","A","B","C","B","D","A","B","C","D","C","B","B","C","A","B","C","B"
-      "scan-type-2.jpg","samples/community/UmarFarootAPS/scans/scan-type-2.jpg","outputs/community/UmarFarootAPS/scans/CheckedOMRs/scan-type-2.jpg","18.333333333333336","0234","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","A","D","","","AD","","","","A","D","","","","","","","D","A","","D","","A","","D","","","","A","","","C","","","D","","","A","","","","D","","C","","A","","C","","D","B","B","","","A","","D","","","","D","","","","","A","D","","","B","","","D","","","A","","","D","","","","","","D","","","","A","D","","","A","","B","","D","","","","C","C","D","D","A","","D","","A","D","","","D","","B","D","","","D","","D","B","","","","D","","A","","","","D","","B","","","","","","D","","","A","","","A","","D","","","D"
+      "scan-type-2.jpg","samples/community/UmarFarootAPS/scans/scan-type-2.jpg","outputs/community/UmarFarootAPS/scans/CheckedOMRs/scan-type-2.jpg","20.0","0234","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","C","D","C","B","A","B","A","D","","","AD","","","","A","D","","","","","","","D","A","","D","","A","","D","","","","A","","","C","","","D","","","A","","","","D","","C","","A","","C","","D","B","B","","","A","","D","","","","D","","","","","A","D","","","B","","","D","","","A","","","D","","","","","","D","","","","A","D","","","A","","B","","D","","","","C","C","D","D","A","","D","","A","D","","","D","","B","D","","","D","","D","B","","","","D","","A","","","","D","","B","","","","","","D","","","A","","","A","","D","","","D"
 
     ''',
   })
@@ -148,8 +183,8 @@
     ''',
     'AdrianSample/Results/Results_05AM.csv': '''
       "file_id","input_path","output_path","score","q1","q2","q3","q4","q5"
-      "adrian_omr.png","samples/sample2/AdrianSample/adrian_omr.png","outputs/sample2/AdrianSample/CheckedOMRs/adrian_omr.png","5.0","B","E","A","C","B"
-      "adrian_omr_2.png","samples/sample2/AdrianSample/adrian_omr_2.png","outputs/sample2/AdrianSample/CheckedOMRs/adrian_omr_2.png","3.0","C","E","A","B","B"
+      "adrian_omr.png","samples/sample2/AdrianSample/adrian_omr.png","outputs/sample2/AdrianSample/CheckedOMRs/adrian_omr.png","0","B","E","A","C","B"
+      "adrian_omr_2.png","samples/sample2/AdrianSample/adrian_omr_2.png","outputs/sample2/AdrianSample/CheckedOMRs/adrian_omr_2.png","0","C","E","A","B","B"
 
     ''',
   })

src/tests/test_all_samples.py

@@ -42,6 +42,16 @@ def extract_sample_outputs(output_dir):
     return sample_outputs
 
 
+def test_run_answer_key_using_csv(mocker, snapshot):
+    sample_outputs = run_sample(mocker, "answer-key/using-csv")
+    assert snapshot == sample_outputs
+
+
+def test_run_answer_key_weighted_answers(mocker, snapshot):
+    sample_outputs = run_sample(mocker, "answer-key/weighted-answers")
+    assert snapshot == sample_outputs
+
+
 def test_run_sample1(mocker, snapshot):
     sample_outputs = run_sample(mocker, "sample1")
     assert snapshot == sample_outputs