Skip to content

Commit

Permalink
Fail loudly if new benchmark has no output.
Browse files Browse the repository at this point in the history
When --make_benchmarks is on, a misconfigured / buggy test program might
fail to produce any output. If this happens, the regression testing
script will continue executing and eventually crash when it can't find
the .status file that it is expecting to be there. This produces an
unintuitive backtrace that does not make clear the root cause of the
problem.

This PR fixes the above by failing with a clear error message as soon as
it can be determined that output from the test program cannot be found.
  • Loading branch information
n8xm committed Apr 17, 2024
1 parent 93ddfb1 commit db67b6b
Showing 1 changed file with 11 additions and 5 deletions.
16 changes: 11 additions & 5 deletions regtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,15 +48,15 @@ def find_build_dirs(tests):
last_safe = False

for obj in tests:

# keep track of the build directory and which source tree it is
# in (e.g. the extra build dir)

# first find the list of unique build directories
dir_pair = (obj.buildDir, obj.extra_build_dir)
if build_dirs.count(dir_pair) == 0:
build_dirs.append(dir_pair)

# re-make all problems that specify an extra compile argument,
# and the test that comes after, just to make sure that any
# unique build commands are seen.
Expand All @@ -66,9 +66,9 @@ def find_build_dirs(tests):
obj.reClean = 0
else:
last_safe = True

return build_dirs

def cmake_setup(suite):
"Setup for cmake"

Expand Down Expand Up @@ -1177,11 +1177,14 @@ def test_suite(argv):
# archive (or delete) the output
#----------------------------------------------------------------------
suite.log.log("archiving the output...")
match_count = 0
for pfile in os.listdir(output_dir):

if (os.path.isdir(pfile) and
re.match(f"{test.name}.*_(plt|chk)[0-9]+", pfile)):

match_count += 1

if suite.purge_output == 1 and not pfile == output_file:

# delete the plt/chk file
Expand All @@ -1206,6 +1209,9 @@ def test_suite(argv):
except OSError:
suite.log.warn(f"unable to remove {pfile}")

if (match_count == 0):
suite.log.fail("ERROR: test output could not be found!")


#----------------------------------------------------------------------
# write the report for this test
Expand Down

0 comments on commit db67b6b

Please sign in to comment.