Skip to content

Commit

Permalink
Fixes failure and adds warnings
Browse files Browse the repository at this point in the history 
- Fixes the failure output for some cases where failed variables were to be output before being defined
- Updates the template to test whether fnrm is within 100*ftol
  - If it is, we're close but not converged. Issue warning from pytest
  - If not, fail as normal
- For warnings to propagate from subprocess, use pytest-isolate instead of pytest-forked
  - Out-of-the-box fix for using pytest-xdist
  - Impacts performance, but the additional information conveyed is worth the tradeoff
  • Loading branch information
@holm10
holm10 committed Dec 11, 2023
1 parent e20258e commit 640163a
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 47 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ jobs:
python -m pip install --upgrade pip
python -m pip install forthon mppl numpy h5py
python -m pip install flake8
python -m pip install pytest-forked pytest-xdist
python -m pip install pytest-isolate pytest-xdist
- name: Build UEDGE
run: |
python setup.py build
python setup.py install
- name: Test with pytest
run: |
# Create a clean UEDGE instance for every test
pytest --forked --tb=native pytests/pytests
pytest --isolate --tb=native pytests/pytests
2 changes: 1 addition & 1 deletion pytests/runtest
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#! /bin/bash

pytest --forked --tb=native pytests
pytest --isolate --tb=native pytests

97 changes: 53 additions & 44 deletions pytests/templates/tests/uedge_tst_template.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ def test_reference(self, epsilon=1e-5):
from copy import deepcopy
from numpy import ndarray, isclose, where, mod
import importlib.util
from warnings import warn
# Then, we can import the UEDGE case setup
inputspec = importlib.util.spec_from_file_location("input", "input.py")
inputspec.loader.exec_module(importlib.util.module_from_spec(inputspec))
Expand Down Expand Up @@ -113,52 +114,60 @@ def print_itroub(refs):
print('Returned fnrm:'.ljust(30), fnrm)
print('Reference fnrm:'.ljust(30), reffnrm)
print_itroub(refs)
print('Failed equation(s):')
for setupkey in ['ni', 'up', 'te', 'ti', 'ng', 'tg', 'phi']:
if setupkey in refs:
match, fnrm, reffnrm = matches(f, refs[setupkey], epsilon)
if match:
continue
elif isinstance(refs[setupkey][f'is{setupkey}on'][()], ndarray):
species = refs['casesetup/nisp'][()]*(setupkey == 'ni') +\
refs['casesetup/nusp'][()]*(setupkey == 'up') +\
refs['casesetup/ngsp'][()]*(setupkey in ['ng', 'tg'])
# Loop through all variables requested
passed, failed = [], []
for var in self.variables():
if isinstance(getdata(var), ndarray):
close = (not False in isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon))
if close:
passed.append(var)
else:
if len(getdata(var).shape) != 3:
failed.append(var)
# See if fnrm is a warning or fail
if matches(f, defref, epsilon=1e2*epsilon)[0]:
warn(UserWarning("Fnrms outside of tolerances, but similar"))
else:
print('Failed equation(s):')
for setupkey in ['ni', 'up', 'te', 'ti', 'ng', 'tg', 'phi']:
if setupkey in refs:
match, fnrm, reffnrm = matches(f, refs[setupkey], epsilon)
if match:
continue
elif isinstance(refs[setupkey][f'is{setupkey}on'][()], ndarray):
species = refs['casesetup/nisp'][()]*(setupkey == 'ni') +\
refs['casesetup/nusp'][()]*(setupkey == 'up') +\
refs['casesetup/ngsp'][()]*(setupkey in ['ng', 'tg'])
# Loop through all variables requested
passed, failed = [], []
for var in self.variables():
if isinstance(getdata(var), ndarray):
close = (not False in isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon))
if close:
passed.append(var)
else:
for i in range(getdata(var).shape[-1]):
if not False in isclose(getdata(var)[:,:,i], refs[f'{setupkey}'][var][:,:,i], atol=0.0, rtol=epsilon):
passed.append(f'{var}[{i}]')
else:
failed.append(f'{var}[{i}]')
else:
if isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon):
passed.append(var)
if len(getdata(var).shape) != 3:
failed.append(var)
else:
for i in range(getdata(var).shape[-1]):
if not False in isclose(getdata(var)[:,:,i], refs[f'{setupkey}'][var][:,:,i], atol=0.0, rtol=epsilon):
passed.append(f'{var}[{i}]')
else:
failed.append(f'{var}[{i}]')
else:
failed.append(var)
failindex = []
for s in range(species):
match, fnrm, reffnrm = matches(f, refs[f'{setupkey}-{s}'], epsilon)
if not match:
failindex.append(s)
print(' - {} for indices: {}'.format(setupkey, str(failindex)[1:-1]))
else:
print(f' - {setupkey}')
print(' - Failed variables: {}'.format(str(failed)[1:-1].replace("'",'')))
# Turn output back on
try:
com.iprint = 1
except:
bbb.iprint = 1
assert False
if isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon):
passed.append(var)
else:
failed.append(var)
failindex = []
for s in range(species):
match, fnrm, reffnrm = matches(f, refs[f'{setupkey}-{s}'], epsilon)
if not match:
failindex.append(s)
print(' - {} for indices: {}'.format(setupkey, str(failindex)[1:-1]))
print(' - Failed variables: {}'.format(str(failed)[1:-1].replace("'",'')))
else:
print(f' - {setupkey}')
print(' - Failed variables: {}'.format(str(failed)[1:-1].replace("'",'')))
# Turn output back on
try:
com.iprint = 1
except:
bbb.iprint = 1
# TODO: Raise warning only if match within 1e-2/1e-3, fail if
# outside 1e-5? Then, tests will pass with warnings if there
# are discrepancies that can be attributed to numerical errors.
assert False



Expand Down

0 comments on commit 640163a

Please sign in to comment.