diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 76adc16e..f1561712 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: python -m pip install --upgrade pip python -m pip install forthon mppl numpy h5py python -m pip install flake8 - python -m pip install pytest-forked pytest-xdist + python -m pip install pytest-isolate pytest-xdist - name: Build UEDGE run: | python setup.py build @@ -32,4 +32,4 @@ jobs: - name: Test with pytest run: | # Create a clean UEDGE instance for every test - pytest --forked --tb=native pytests/pytests + pytest --isolate --tb=native pytests/pytests diff --git a/pytests/runtest b/pytests/runtest index da02eafb..3e3d74f4 100755 --- a/pytests/runtest +++ b/pytests/runtest @@ -1,4 +1,4 @@ #! /bin/bash -pytest --forked --tb=native pytests +pytest --isolate --tb=native pytests diff --git a/pytests/templates/tests/uedge_tst_template.py b/pytests/templates/tests/uedge_tst_template.py index 54294bce..d8be21af 100644 --- a/pytests/templates/tests/uedge_tst_template.py +++ b/pytests/templates/tests/uedge_tst_template.py @@ -25,6 +25,7 @@ def test_reference(self, epsilon=1e-5): from copy import deepcopy from numpy import ndarray, isclose, where, mod import importlib.util + from warnings import warn # Then, we can import the UEDGE case setup inputspec = importlib.util.spec_from_file_location("input", "input.py") inputspec.loader.exec_module(importlib.util.module_from_spec(inputspec)) @@ -113,52 +114,60 @@ def print_itroub(refs): print('Returned fnrm:'.ljust(30), fnrm) print('Reference fnrm:'.ljust(30), reffnrm) print_itroub(refs) - print('Failed equation(s):') - for setupkey in ['ni', 'up', 'te', 'ti', 'ng', 'tg', 'phi']: - if setupkey in refs: - match, fnrm, reffnrm = matches(f, refs[setupkey], epsilon) - if match: - continue - elif isinstance(refs[setupkey][f'is{setupkey}on'][()], ndarray): - species = refs['casesetup/nisp'][()]*(setupkey == 'ni') +\ - refs['casesetup/nusp'][()]*(setupkey == 'up') +\ - refs['casesetup/ngsp'][()]*(setupkey in ['ng', 'tg']) - # Loop through all variables requested - passed, failed = [], [] - for var in self.variables(): - if isinstance(getdata(var), ndarray): - close = (not False in isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon)) - if close: - passed.append(var) - else: - if len(getdata(var).shape) != 3: - failed.append(var) + # See if fnrm is a warning or fail + if matches(f, defref, epsilon=1e2*epsilon)[0]: + warn(UserWarning("Fnrms outside of tolerances, but similar")) + else: + print('Failed equation(s):') + for setupkey in ['ni', 'up', 'te', 'ti', 'ng', 'tg', 'phi']: + if setupkey in refs: + match, fnrm, reffnrm = matches(f, refs[setupkey], epsilon) + if match: + continue + elif isinstance(refs[setupkey][f'is{setupkey}on'][()], ndarray): + species = refs['casesetup/nisp'][()]*(setupkey == 'ni') +\ + refs['casesetup/nusp'][()]*(setupkey == 'up') +\ + refs['casesetup/ngsp'][()]*(setupkey in ['ng', 'tg']) + # Loop through all variables requested + passed, failed = [], [] + for var in self.variables(): + if isinstance(getdata(var), ndarray): + close = (not False in isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon)) + if close: + passed.append(var) else: - for i in range(getdata(var).shape[-1]): - if not False in isclose(getdata(var)[:,:,i], refs[f'{setupkey}'][var][:,:,i], atol=0.0, rtol=epsilon): - passed.append(f'{var}[{i}]') - else: - failed.append(f'{var}[{i}]') - else: - if isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon): - passed.append(var) + if len(getdata(var).shape) != 3: + failed.append(var) + else: + for i in range(getdata(var).shape[-1]): + if not False in isclose(getdata(var)[:,:,i], refs[f'{setupkey}'][var][:,:,i], atol=0.0, rtol=epsilon): + passed.append(f'{var}[{i}]') + else: + failed.append(f'{var}[{i}]') else: - failed.append(var) - failindex = [] - for s in range(species): - match, fnrm, reffnrm = matches(f, refs[f'{setupkey}-{s}'], epsilon) - if not match: - failindex.append(s) - print(' - {} for indices: {}'.format(setupkey, str(failindex)[1:-1])) - else: - print(f' - {setupkey}') - print(' - Failed variables: {}'.format(str(failed)[1:-1].replace("'",''))) - # Turn output back on - try: - com.iprint = 1 - except: - bbb.iprint = 1 - assert False + if isclose(getdata(var), refs[f'{setupkey}'][var][()], atol=0.0, rtol=epsilon): + passed.append(var) + else: + failed.append(var) + failindex = [] + for s in range(species): + match, fnrm, reffnrm = matches(f, refs[f'{setupkey}-{s}'], epsilon) + if not match: + failindex.append(s) + print(' - {} for indices: {}'.format(setupkey, str(failindex)[1:-1])) + print(' - Failed variables: {}'.format(str(failed)[1:-1].replace("'",''))) + else: + print(f' - {setupkey}') + print(' - Failed variables: {}'.format(str(failed)[1:-1].replace("'",''))) + # Turn output back on + try: + com.iprint = 1 + except: + bbb.iprint = 1 + # TODO: Raise warning only if match within 1e-2/1e-3, fail if + # outside 1e-5? Then, tests will pass with warnings if there + # are discrepancies that can be attributed to numerical errors. + assert False