contest: subcases: handle retry cases

In case of failure, some tests are restarted. That's great, but when
parsing the subcases (nested tests), the previous results were
overridden. Also, when displaying the results, the 'retry' field from
the main test was re-used in each subcases: quite confusing for the
reader.

Now, in case of retry, the previous 'results' list will be modified to
add a new 'retry' entry for each test that has been re-validated. If the
previous test with the same name cannot be found, that could be because
there was major issue before and some subcases have not been executed
(or the names are not fixed). In this case, a new entry with a failed
first attempt will be added to the list. When querying the subcases
results, the new 'retry' entry will be used, or none if it is not there.

Signed-off-by: Matthieu Baerts (NGI0) <[email protected]>

Author: matttbe

contest/backend/query.py

@@ -63,6 +63,9 @@ def result_as_l2(raw):
                 del data["results"]
                 if "time" in data:
                     del data["time"]
+                # in case of retry, the subtest might not have been re-executed
+                if "retry" in data:
+                    del data["retry"]
                 data |= case
                 data["test"] = l1["test"] + '.' + case["test"]
                 flat.append(data)

contest/remote/vmksft-p.py

@@ -69,7 +69,7 @@ def get_prog_list(vm, targets):
     return [(e.split(":")[0].strip(), e.split(":")[1].strip()) for e in targets]
 
 
-def _parse_nested_tests(full_run):
+def _parse_nested_tests(full_run, prev_results):
     tests = []
     nested_tests = False
 
@@ -110,8 +110,22 @@ def _parse_nested_tests(full_run):
 
         r['result'] = result
 
-        tests.append(r)
+        if prev_results is not None:
+            found = False
+            for entry in prev_results:
+                if entry['test'] == r['test']:
+                    entry['retry'] = result
+                    found = True
+                    break
+            if not found:
+                # the first run didn't validate this test: mark 1st attempt as fail
+                r['result'] = 'fail'
+                r['retry'] = result
+                prev_results.append(r)
+        else:
+            tests.append(r)
 
+    # return an empty list when there are prev results: no replacement needed
     return tests
 
 def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue):
@@ -195,8 +209,13 @@ def _vm_thread(config, results_path, thr_id, hard_stop, in_queue, out_queue):
                 outcome['crashes'] = crashes
 
         if config.getboolean('ksft', 'nested_tests', fallback=False):
+            if is_retry:
+                prev_results = outcome['results'] if 'results' in outcome else []
+            else:
+                prev_results = None
+
             # this will only parse nested tests inside the TAP comments
-            nested_tests = _parse_nested_tests(vm.log_out)
+            nested_tests = _parse_nested_tests(vm.log_out, prev_results)
             if nested_tests:
                 outcome['results'] = nested_tests