From 74bcb07615571b156c794841ca2615fb72ac5c54 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Sun, 16 Jun 2024 10:40:13 -0500 Subject: [PATCH 1/6] ci: update workflows --- .github/workflows/{pymake-gcc.yml => ci.yml} | 36 ++++++-- .../workflows/{main.yml => pymake-intel.yml} | 2 +- .gitignore | 1 + autotest/test_mflgr.py | 34 ------- autotest/test_mfnwt.py | 91 ------------------- autotest/test_triangle_makefile.py | 69 -------------- pixi.toml | 3 +- 7 files changed, 29 insertions(+), 207 deletions(-) rename .github/workflows/{pymake-gcc.yml => ci.yml} (63%) rename .github/workflows/{main.yml => pymake-intel.yml} (98%) delete mode 100644 autotest/test_mflgr.py delete mode 100644 autotest/test_mfnwt.py delete mode 100644 autotest/test_triangle_makefile.py diff --git a/.github/workflows/pymake-gcc.yml b/.github/workflows/ci.yml similarity index 63% rename from .github/workflows/pymake-gcc.yml rename to .github/workflows/ci.yml index d514ba78..32f6eca9 100644 --- a/.github/workflows/pymake-gcc.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: pymake gcc build +name: pymake continuous integration on: schedule: @@ -19,7 +19,7 @@ concurrency: cancel-in-progress: true jobs: - pymakeCI-os-gcc: + pymakeCI-os-compiler: name: pymake CI gcc on different OSs runs-on: ${{ matrix.os }} strategy: @@ -27,12 +27,16 @@ jobs: matrix: include: # test latest gcc and python - - {os: ubuntu-latest, gcc: 13} - - {os: windows-latest, gcc: 13} - - {os: macos-latest, gcc: 13} + - {os: ubuntu-latest, FC: gcc, FC_V: 13} + - {os: windows-latest, FC: gcc, FC_V: 13} + - {os: macos-latest, FC: gcc, FC_V: 13} # test latest python and previous gcc - - {os: ubuntu-latest, gcc: 12} - - {os: ubuntu-latest, gcc: 11} + - {os: ubuntu-latest, FC: gcc, FC_V: 12} + - {os: ubuntu-latest, FC: gcc, FC_V: 11} + # test latest python and intel + - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} + - {os: windows-latest, FC: intel-classic, FC_V: 2021.7} + - {os: macos-13, FC: intel-classic, FC_V: 2021.7} defaults: run: shell: bash @@ -41,6 +45,11 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 + - uses: maxim-lobanov/setup-xcode@v1 + if: ${{ (runner.os == 'macOS') && (matrix.FC == 'intel-classic') }} + with: + xcode-version: "14.3.1" + - name: Setup Graphviz on Linux if: runner.os == 'Linux' uses: ts-graphviz/setup-graphviz@v2 @@ -56,11 +65,11 @@ jobs: run: | pixi run postinstall - - name: Setup GNU Fortran + - name: Setup ${{ env.FC }} ${{ env.FC_V }} uses: fortran-lang/setup-fortran@main with: - compiler: gcc - version: ${{ matrix.gcc }} + compiler: ${{ matrix.FC }} + version: ${{ matrix.FC_V }} - name: Download examples for pytest runs run: | @@ -81,6 +90,13 @@ jobs: run: | pixi run autotest-Windows + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} + path: ./autotest/.failed + - name: Print coverage report before upload run: | pixi run coverage-report diff --git a/.github/workflows/main.yml b/.github/workflows/pymake-intel.yml similarity index 98% rename from .github/workflows/main.yml rename to .github/workflows/pymake-intel.yml index 8c901ed5..fe4f0fbc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/pymake-intel.yml @@ -1,4 +1,4 @@ -name: pymake continuous integration +name: pymake continuous integration intel on: schedule: diff --git a/.gitignore b/.gitignore index f700a606..66ff0ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ target/ # files in the autotest directory autotest/temp*/ +autotest/.failed/ autotest/*.json autotest/code.md mod_temp/ diff --git a/autotest/test_mflgr.py b/autotest/test_mflgr.py deleted file mode 100644 index 1de8c968..00000000 --- a/autotest/test_mflgr.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - -import pytest - -import pymake - - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> Path: - target = "mflgr" - return module_tmpdir / target - - -@pytest.fixture(scope="module") -def prog_dict(target) -> dict: - return pymake.usgs_program_data.get_target(target) - - -@pytest.fixture(scope="module") -def workspace(module_tmpdir, prog_dict) -> Path: - return module_tmpdir / prog_dict.dirname - - -def compile_code(ws, exe): - return pymake.build_apps( - str(exe), download_dir=ws, appdir=ws, verbose=True - ) - - -@pytest.mark.base -def test_compile(module_tmpdir, target): - assert ( - compile_code(module_tmpdir, target) == 0 - ), f"could not compile {target}" diff --git a/autotest/test_mfnwt.py b/autotest/test_mfnwt.py deleted file mode 100644 index f7cb8013..00000000 --- a/autotest/test_mfnwt.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import time -from pathlib import Path - -import pytest -from modflow_devtools.misc import set_dir - -import pymake - - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> str: - target = "mfnwt" - return module_tmpdir / target - - -@pytest.fixture(scope="module") -def prog_data(target) -> dict: - return pymake.usgs_program_data.get_target(target.name) - - -@pytest.fixture(scope="module") -def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - pm.makefile = True - pm.makefiledir = str(module_tmpdir) - pm.inplace = True - pm.dryrun = False - pm.verbose = True - yield pm - pm.finalize() - - -def build_with_makefile(ws): - success = True - with set_dir(ws): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {target} with makefile") - os.system("make clean") - - # build MODFLOW-NWT with makefile - print(f"build {target} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - errmsg = f"{target} created by makefile does not exist." - if sys.platform.lower() == "win32" and pm.fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that MODFLOW-NWT was made - else: - success = os.path.isfile(target) - else: - errmsg = "makefile does not exist" - - assert success, errmsg - - -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" - - -@pytest.mark.dependency(name="makefile", depends=["build"]) -@pytest.mark.base -def test_makefile(workspace): - build_with_makefile(workspace) diff --git a/autotest/test_triangle_makefile.py b/autotest/test_triangle_makefile.py deleted file mode 100644 index 45579d87..00000000 --- a/autotest/test_triangle_makefile.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import sys - -import flopy -import pytest - -import pymake - - -@pytest.mark.base -def test_pymake_makefile(module_tmpdir): - target = "triangle" - pm = pymake.Pymake(verbose=True) - pm.makefile = True - pm.makeclean = True - # pm.cc = "gcc" - - if sys.platform.lower() == "win32": - if pm.cc == "icl": - return - target += ".exe" - - # get current directory - cwd = os.getcwd() - - # change to working directory so triangle download directory is - # a subdirectory in the working directory - os.chdir(module_tmpdir) - - # build triangle and makefile - assert ( - pymake.build_apps(target, clean=False, pymake_object=pm) == 0 - ), f"could not build {target}" - - if os.path.isfile(os.path.join(module_tmpdir, "makefile")): - print("cleaning with GNU make") - # clean prior to make - print(f"clean {target} with makefile") - success, _ = flopy.run_model( - "make", - None, - cargs="clean", - model_ws=module_tmpdir, - report=True, - normal_msg="rm -rf ./triangle", - silent=False, - ) - - # build triangle with makefile - if success: - print(f"build {target} with makefile") - success, _ = flopy.run_model( - "make", - None, - model_ws=module_tmpdir, - report=True, - normal_msg="cc -O2 -o triangle ./obj_temp/triangle.o", - silent=False, - ) - - # finalize Pymake object - pm.finalize() - - # return to starting directory - os.chdir(cwd) - - assert os.path.isfile( - os.path.join(module_tmpdir, target) - ), f"could not build {target} with makefile" diff --git a/pixi.toml b/pixi.toml index 589bea5c..63f34b44 100644 --- a/pixi.toml +++ b/pixi.toml @@ -34,9 +34,8 @@ postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version- # format check-lint = "ruff check ." -fix-lint = "ruff check . --fix" check-format = "ruff format . --check" -fix-format = "ruff format ." +fix-style = "ruff check . --fix; ruff format ." # build test = "meson test --verbose --no-rebuild -C" From 133ab8365163a4c585feb40ea4c8a09cf23d819f Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Sun, 16 Jun 2024 11:55:33 -0500 Subject: [PATCH 2/6] * update request ci to use pixi --- .github/workflows/ci.yml | 89 ++++++++++++++++++++++----- .github/workflows/pymake-requests.yml | 32 +++++----- autotest/test_build.py | 15 +++-- autotest/test_cli_cmds.py | 55 +++++++++-------- autotest/test_gridgen.py | 4 +- autotest/test_mf2005.py | 4 +- autotest/test_mf6.py | 40 +----------- autotest/test_mfusg.py | 6 +- autotest/test_misc_programs.py | 24 -------- autotest/test_mp6.py | 4 +- autotest/test_mp7.py | 4 +- autotest/test_mt3d.py | 8 +-- autotest/test_seawat.py | 4 +- pixi.toml | 2 + 14 files changed, 148 insertions(+), 143 deletions(-) delete mode 100644 autotest/test_misc_programs.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32f6eca9..7da69910 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,8 +19,9 @@ concurrency: cancel-in-progress: true jobs: - pymakeCI-os-compiler: + pymake-schedule: name: pymake CI gcc on different OSs + if: ${{ github.event_name == 'schedule' }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -28,15 +29,80 @@ jobs: include: # test latest gcc and python - {os: ubuntu-latest, FC: gcc, FC_V: 13} + - {os: macos-latest, FC: gcc, FC_V: 13} - {os: windows-latest, FC: gcc, FC_V: 13} + # test latest python and intel + - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} + - {os: macos-13, FC: intel-classic, FC_V: 2021.7} + - {os: windows-2019, FC: intel-classic, FC_V: 2021.7} + defaults: + run: + shell: bash + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - uses: maxim-lobanov/setup-xcode@v1 + if: ${{ (runner.os == 'macOS') && (matrix.FC == 'intel-classic') }} + with: + xcode-version: "14.3.1" + + - name: Setup Graphviz on Linux + if: runner.os == 'Linux' + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.19.1 + manifest-path: "pixi.toml" + + - name: pixi post-install + working-directory: pymake + run: | + pixi run postinstall + + - name: Setup ${{ matrix.FC }} ${{ matrix.FC_V }} on ${{ matrix.os }} + uses: fortran-lang/setup-fortran@main + with: + compiler: ${{ matrix.FC }} + version: ${{ matrix.FC_V }} + + - name: Install make + if: runner.os == 'Windows' + run: choco install make + + - name: Run scheduled tests + run: | + pixi run autotest-schedule + + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-schedule-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} + path: ./autotest/.failed + + + pymake-os-compiler: + name: pymake CI gcc on different OSs + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # test latest gcc and python + - {os: ubuntu-latest, FC: gcc, FC_V: 13} - {os: macos-latest, FC: gcc, FC_V: 13} + - {os: windows-latest, FC: gcc, FC_V: 13} + # test latest python and intel-classic + - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} + - {os: macos-13, FC: intel-classic, FC_V: 2021.7} + - {os: windows-2019, FC: intel-classic, FC_V: 2021.7} # test latest python and previous gcc - {os: ubuntu-latest, FC: gcc, FC_V: 12} - {os: ubuntu-latest, FC: gcc, FC_V: 11} - # test latest python and intel - - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} - - {os: windows-latest, FC: intel-classic, FC_V: 2021.7} - - {os: macos-13, FC: intel-classic, FC_V: 2021.7} defaults: run: shell: bash @@ -65,7 +131,7 @@ jobs: run: | pixi run postinstall - - name: Setup ${{ env.FC }} ${{ env.FC_V }} + - name: Setup ${{ matrix.FC }} ${{ matrix.FC_V }} on ${{ matrix.os }} uses: fortran-lang/setup-fortran@main with: compiler: ${{ matrix.FC }} @@ -96,14 +162,5 @@ jobs: with: name: failed-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} path: ./autotest/.failed - - - name: Print coverage report before upload - run: | - pixi run coverage-report - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./autotest/coverage.xml + diff --git a/.github/workflows/pymake-requests.yml b/.github/workflows/pymake-requests.yml index 514d0e1e..472efb90 100644 --- a/.github/workflows/pymake-requests.yml +++ b/.github/workflows/pymake-requests.yml @@ -30,32 +30,32 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.8.1 with: - python-version: "3.12" + pixi-version: v0.19.1 + manifest-path: "pixi.toml" - - name: Install python packages + - name: pixi post-install + working-directory: pymake run: | - python -m pip install --upgrade pip - pip install ".[test]" + pixi run postinstall - name: Run pytest - working-directory: ./autotest run: | - pytest -v -n=auto -m requests --durations=0 --cov=pymake --cov-report=xml + pixi run autotest-request - - name: Run scheduled tests - if: ${{ github.event_name == 'schedule' }} - working-directory: ./autotest - run: | - pytest -v -m="schedule" --durations=0 --cov=pymake --cov-report=xml + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-requests + path: ./autotest/.failed - name: Print coverage report before upload - working-directory: ./autotest run: | - coverage report - + pixi run coverage-report + - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 env: diff --git a/autotest/test_build.py b/autotest/test_build.py index 1e02c2e8..85a8b5df 100644 --- a/autotest/test_build.py +++ b/autotest/test_build.py @@ -12,11 +12,6 @@ RERUNS = 3 targets = pymake.usgs_program_data.get_keys(current=True) -targets_make = [ - t - for t in targets - if t not in ("libmf6", "gridgen", "mf2000", "swtv4", "mflgr") -] test_ostag = get_ostag() test_fc_env = os.environ.get("FC") if "win" in test_ostag: @@ -27,6 +22,15 @@ meson_exclude = ("sutra",) targets_meson = [t for t in targets if t not in meson_exclude] +if "win" in test_ostag and test_fc_env in ("ifort",): + targets_make = [] +else: + targets_make = [ + t + for t in targets + if t not in ("libmf6", "gridgen", "mf2000", "swtv4", "mflgr") + ] + def build_with_makefile(target, path, fc): success = True @@ -103,7 +107,6 @@ def test_meson_build(function_tmpdir, target: str) -> None: @pytest.mark.base @flaky(max_runs=RERUNS) -@pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize("target", targets_make) def test_makefile_build(function_tmpdir, target: str) -> None: pm = pymake.Pymake(verbose=True) diff --git a/autotest/test_cli_cmds.py b/autotest/test_cli_cmds.py index f31e0a37..6adb0894 100644 --- a/autotest/test_cli_cmds.py +++ b/autotest/test_cli_cmds.py @@ -46,43 +46,46 @@ def run_cli_cmd(cmd: list) -> None: @pytest.mark.base @pytest.mark.parametrize("target", targets) def test_make_program(function_tmpdir, target: str) -> None: - cmd = [ - "make-program", - target, - "--appdir", - str(function_tmpdir), - "--verbose", - ] - run_cli_cmd(cmd) + with set_dir(function_tmpdir): + cmd = [ + "make-program", + target, + "--appdir", + str(function_tmpdir), + "--verbose", + ] + run_cli_cmd(cmd) @flaky(max_runs=RERUNS) @pytest.mark.dependency(name="make_program") @pytest.mark.base def test_make_program_double(function_tmpdir) -> None: - cmd = [ - "make-program", - "mf2005", - "--double", - "--verbose", - "--appdir", - str(function_tmpdir), - ] - run_cli_cmd(cmd) + with set_dir(function_tmpdir): + cmd = [ + "make-program", + "mf2005", + "--double", + "--verbose", + "--appdir", + str(function_tmpdir), + ] + run_cli_cmd(cmd) @pytest.mark.dependency(name="make_program_all") @pytest.mark.schedule def test_make_program_all(module_tmpdir) -> None: - cmd = [ - "make-program", - ":", - "--appdir", - str(module_tmpdir / "all"), - "--verbose", - "--dryrun", - ] - run_cli_cmd(cmd) + with set_dir(module_tmpdir): + cmd = [ + "make-program", + ":", + "--appdir", + str(module_tmpdir / "all"), + "--verbose", + "--dryrun", + ] + run_cli_cmd(cmd) @flaky(max_runs=RERUNS) diff --git a/autotest/test_gridgen.py b/autotest/test_gridgen.py index 1c693126..8f6194b5 100644 --- a/autotest/test_gridgen.py +++ b/autotest/test_gridgen.py @@ -59,14 +59,14 @@ def run_gridgen(cmd, ws, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" diff --git a/autotest/test_mf2005.py b/autotest/test_mf2005.py index dbf361ea..51887fa1 100644 --- a/autotest/test_mf2005.py +++ b/autotest/test_mf2005.py @@ -43,14 +43,14 @@ def run_mf2005(namefile, ws, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target}" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" diff --git a/autotest/test_mf6.py b/autotest/test_mf6.py index b9470f64..3ae73c1c 100644 --- a/autotest/test_mf6.py +++ b/autotest/test_mf6.py @@ -100,14 +100,14 @@ def build_with_makefile(pm, workspace, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @@ -118,39 +118,3 @@ def test_compile(pm, target): def test_mf6(ws, target): success, _ = flopy.run_model(target, None, model_ws=ws, silent=False) assert success, f"could not run {ws}" - - -@pytest.mark.dependency(name="makefile", depends=["build"]) -@pytest.mark.base -def test_makefile(pm, module_tmpdir, target): - assert build_with_makefile( - pm, module_tmpdir, target - ), f"could not compile {target} with makefile" - - -@pytest.mark.dependency(name="shared", depends=["makefile"]) -@pytest.mark.base -def test_sharedobject(pm, module_tmpdir, workspace, target_so, prog_data): - # reconfigure pymake object - pm.target = str(target_so) - pm.appdir = module_tmpdir - pm.srcdir = workspace / prog_data.srcdir - pm.srcdir2 = workspace / "src" - pm.excludefiles = [os.path.join(pm.srcdir2, "mf6.f90")] - pm.makefile = True - pm.makeclean = True - pm.sharedobject = True - pm.inplace = True - pm.dryrun = False - - # build the target - assert pm.build() == 0, f"could not compile {pm.target}" - assert target_so.is_file() - - -@pytest.mark.dependency(name="shared_makefile", depends=["shared", "makefile"]) -@pytest.mark.base -def test_sharedobject_makefile(pm, module_tmpdir, target_so): - assert build_with_makefile( - pm, module_tmpdir, target_so - ), f"could not compile {target_so} with makefile" diff --git a/autotest/test_mfusg.py b/autotest/test_mfusg.py index 0f13c3e3..5488b90d 100644 --- a/autotest/test_mfusg.py +++ b/autotest/test_mfusg.py @@ -68,7 +68,7 @@ def run_mfusg(fn, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regrression def test_download(pm, pm_gsi, module_tmpdir, targets): pm.download_target(targets[0], download_path=module_tmpdir) assert pm.download, f"could not download {targets[0]}" @@ -78,7 +78,7 @@ def test_download(pm, pm_gsi, module_tmpdir, targets): @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, pm_gsi, targets): assert pm.build() == 0, f"could not compile {targets[0]}" assert (targets[0]).is_file() @@ -87,7 +87,7 @@ def test_compile(pm, pm_gsi, targets): assert targets[1].is_file() -@pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.dependency(name="test", depends=["download", "build"]) @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_misc_programs.py b/autotest/test_misc_programs.py deleted file mode 100644 index 93d4e30a..00000000 --- a/autotest/test_misc_programs.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -import pymake - -targets = [ - "crt", - "vs2dt", - "zonbud3", -] - - -@pytest.mark.base -@pytest.mark.parametrize("target", targets) -def test_compile(module_tmpdir, target): - bin_dir = module_tmpdir / "bin" - assert ( - pymake.build_apps( - str(bin_dir / target), - download_dir=str(module_tmpdir), - appdir=str(bin_dir), - verbose=True, - ) - == 0 - ), f"could not compile {target}" diff --git a/autotest/test_mp6.py b/autotest/test_mp6.py index 51c14522..70dfc2b9 100644 --- a/autotest/test_mp6.py +++ b/autotest/test_mp6.py @@ -60,14 +60,14 @@ def update_files(fn, workspace): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" diff --git a/autotest/test_mp7.py b/autotest/test_mp7.py index 24b1c669..5b4265fb 100644 --- a/autotest/test_mp7.py +++ b/autotest/test_mp7.py @@ -143,14 +143,14 @@ def run_modpath7(namefile, mp7_exe, mf2005_exe, mfusg_exe, mf6_exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" diff --git a/autotest/test_mt3d.py b/autotest/test_mt3d.py index f9c2ea0b..fd09c89f 100644 --- a/autotest/test_mt3d.py +++ b/autotest/test_mt3d.py @@ -88,7 +88,7 @@ def run_mt3dusgs(workspace, mt3dms_exe, mfnwt_exe, mf6_exe): @pytest.mark.dependency(name="download_mt3dms") -@pytest.mark.base +@pytest.mark.regression def test_download_mt3dms(pm, module_tmpdir): pm.target = "mt3dms" pm.download_target(pm.target, download_path=module_tmpdir) @@ -96,13 +96,13 @@ def test_download_mt3dms(pm, module_tmpdir): @pytest.mark.dependency(name="build_mt3dms", depends=["download_mt3dms"]) -@pytest.mark.base +@pytest.mark.regression def test_compile_mt3dms(pm): assert pm.build() == 0, f"could not compile {pm.target}" @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.reset(str(target)) pm.download_target(target, download_path=module_tmpdir) @@ -110,7 +110,7 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" diff --git a/autotest/test_seawat.py b/autotest/test_seawat.py index 1dfbd431..d64fc13c 100644 --- a/autotest/test_seawat.py +++ b/autotest/test_seawat.py @@ -76,14 +76,14 @@ def build_seawat_dependency_graphs(src_path, dep_path): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target}" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" diff --git a/pixi.toml b/pixi.toml index 63f34b44..6a802543 100644 --- a/pixi.toml +++ b/pixi.toml @@ -45,6 +45,8 @@ download-examples = {cmd = "python ci_setup.py", cwd = "autotest"} autotest = { cmd = "pytest -v -n auto --dist=loadfile -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } autotest-base = { cmd = "pytest -v -n auto --dist=loadfile -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } autotest-Windows = { cmd = "pytest -v -m='base' --durations 0 --cov=pymake --cov-report=xml --basetemp=$RUNNER_TEMP/pytest_temp --keep-failed .failed", cwd = "autotest" } +autotest-request = { cmd = "pytest -v -n=auto -m='requests' --durations=0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-schedule = { cmd = "pytest -v -m='schedule' --durations=0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } # coverage report coverage-report = { cmd = "coverage report", cwd = "autotest"} From 58ffea0b6e218390532dadb47a5d02d87de250be Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Sun, 16 Jun 2024 12:26:26 -0500 Subject: [PATCH 3/6] * update --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++++++++++----- autotest/test_cli_cmds.py | 1 - autotest/test_mfusg.py | 2 +- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7da69910..a11d0050 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,7 +83,18 @@ jobs: with: name: failed-schedule-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} path: ./autotest/.failed - + + - name: Print coverage report before upload + run: | + pixi run coverage-report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + file: ./autotest/coverage.xml + pymake-os-compiler: name: pymake CI gcc on different OSs @@ -145,8 +156,13 @@ jobs: if: runner.os == 'Windows' run: choco install make - - name: Run pytest on Linux and MacOS - if: runner.os != 'Windows' + - name: test on Linux + if: runner.os == 'Linux' + run: | + pixi run autotest + + - name: test on MacOS + if: runner.os == 'macOS' run: | pixi run autotest-base @@ -154,7 +170,7 @@ jobs: if: runner.os == 'Windows' shell: pwsh run: | - pixi run autotest-Windows + pixi run autotest-base - name: Upload failed test output if: failure() @@ -163,4 +179,14 @@ jobs: name: failed-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} path: ./autotest/.failed - + - name: Print coverage report before upload + run: | + pixi run coverage-report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + file: ./autotest/coverage.xml + diff --git a/autotest/test_cli_cmds.py b/autotest/test_cli_cmds.py index 6adb0894..4e635899 100644 --- a/autotest/test_cli_cmds.py +++ b/autotest/test_cli_cmds.py @@ -83,7 +83,6 @@ def test_make_program_all(module_tmpdir) -> None: "--appdir", str(module_tmpdir / "all"), "--verbose", - "--dryrun", ] run_cli_cmd(cmd) diff --git a/autotest/test_mfusg.py b/autotest/test_mfusg.py index 5488b90d..ab8c3d92 100644 --- a/autotest/test_mfusg.py +++ b/autotest/test_mfusg.py @@ -68,7 +68,7 @@ def run_mfusg(fn, exe): @pytest.mark.dependency(name="download") -@pytest.mark.regrression +@pytest.mark.regression def test_download(pm, pm_gsi, module_tmpdir, targets): pm.download_target(targets[0], download_path=module_tmpdir) assert pm.download, f"could not download {targets[0]}" From e7dcf43650da300e97b0c0b445cb58c365476864 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Sun, 16 Jun 2024 14:11:11 -0500 Subject: [PATCH 4/6] * update --- .github/workflows/ci.yml | 66 ++++++++--------- .github/workflows/pymake-intel.yml | 95 ------------------------ autotest/test_build.py | 86 ++++++++++------------ autotest/test_gridgen.py | 7 +- autotest/test_mf2005.py | 7 +- autotest/test_mf6.py | 38 ++-------- autotest/test_mf6_existing_meson.py | 108 +++++++++++++++------------- autotest/test_mfusg.py | 3 + autotest/test_mp6.py | 7 +- autotest/test_mp7.py | 8 ++- autotest/test_mt3d.py | 12 +++- autotest/test_seawat.py | 8 ++- pixi.toml | 5 +- 13 files changed, 180 insertions(+), 270 deletions(-) delete mode 100644 .github/workflows/pymake-intel.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a11d0050..f9935c63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,9 +19,8 @@ concurrency: cancel-in-progress: true jobs: - pymake-schedule: - name: pymake CI gcc on different OSs - if: ${{ github.event_name == 'schedule' }} + pymake-os-compiler: + name: pymake CI on different OSs with gcc and intel-classic runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -31,10 +30,13 @@ jobs: - {os: ubuntu-latest, FC: gcc, FC_V: 13} - {os: macos-latest, FC: gcc, FC_V: 13} - {os: windows-latest, FC: gcc, FC_V: 13} - # test latest python and intel + # test latest python and intel-classic - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} - {os: macos-13, FC: intel-classic, FC_V: 2021.7} - {os: windows-2019, FC: intel-classic, FC_V: 2021.7} + # test latest python and previous gcc + - {os: ubuntu-latest, FC: gcc, FC_V: 12} + - {os: ubuntu-latest, FC: gcc, FC_V: 11} defaults: run: shell: bash @@ -68,20 +70,36 @@ jobs: with: compiler: ${{ matrix.FC }} version: ${{ matrix.FC_V }} + + - name: Download examples for pytest runs + run: | + pixi run download-examples - name: Install make if: runner.os == 'Windows' run: choco install make - - name: Run scheduled tests + - name: test on Linux + if: runner.os == 'Linux' run: | - pixi run autotest-schedule + pixi run autotest + + - name: test on MacOS + if: runner.os == 'macOS' + run: | + pixi run autotest-base + + - name: test on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + pixi run autotest-base - name: Upload failed test output if: failure() uses: actions/upload-artifact@v4 with: - name: failed-schedule-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} + name: failed-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} path: ./autotest/.failed - name: Print coverage report before upload @@ -95,9 +113,9 @@ jobs: with: file: ./autotest/coverage.xml - - pymake-os-compiler: - name: pymake CI gcc on different OSs + pymake-schedule: + name: pymake scheduled CI different OSs with gcc and intel-classic + if: ${{ github.event_name == 'schedule' }} runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -107,13 +125,10 @@ jobs: - {os: ubuntu-latest, FC: gcc, FC_V: 13} - {os: macos-latest, FC: gcc, FC_V: 13} - {os: windows-latest, FC: gcc, FC_V: 13} - # test latest python and intel-classic + # test latest python and intel - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} - {os: macos-13, FC: intel-classic, FC_V: 2021.7} - {os: windows-2019, FC: intel-classic, FC_V: 2021.7} - # test latest python and previous gcc - - {os: ubuntu-latest, FC: gcc, FC_V: 12} - - {os: ubuntu-latest, FC: gcc, FC_V: 11} defaults: run: shell: bash @@ -147,36 +162,20 @@ jobs: with: compiler: ${{ matrix.FC }} version: ${{ matrix.FC_V }} - - - name: Download examples for pytest runs - run: | - pixi run download-examples - name: Install make if: runner.os == 'Windows' run: choco install make - - name: test on Linux - if: runner.os == 'Linux' - run: | - pixi run autotest - - - name: test on MacOS - if: runner.os == 'macOS' - run: | - pixi run autotest-base - - - name: test on Windows - if: runner.os == 'Windows' - shell: pwsh + - name: Run scheduled tests run: | - pixi run autotest-base + pixi run autotest-schedule - name: Upload failed test output if: failure() uses: actions/upload-artifact@v4 with: - name: failed-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} + name: failed-schedule-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} path: ./autotest/.failed - name: Print coverage report before upload @@ -190,3 +189,4 @@ jobs: with: file: ./autotest/coverage.xml + diff --git a/.github/workflows/pymake-intel.yml b/.github/workflows/pymake-intel.yml deleted file mode 100644 index fe4f0fbc..00000000 --- a/.github/workflows/pymake-intel.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: pymake continuous integration intel - -on: - schedule: - - cron: '0 7 * * *' # run at 7 AM UTC every day - push: - paths-ignore: - - 'README.md' - - 'docs/*.md' - pull_request: - branches: - - master - - develop - paths-ignore: - - 'README.md' - - 'docs/*.md' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -jobs: - - pymakeCI-os-intel: - name: pymake CI intel on different OSs - runs-on: ${{ matrix.os }} - env: - FC: intel-classic - FC_V: "2021.7" - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-13, windows-2019] - defaults: - run: - shell: bash - - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - uses: maxim-lobanov/setup-xcode@v1 - if: runner.os == 'macOS' - with: - xcode-version: "14.3.1" - - - name: Setup ${{ env.FC }} ${{ env.FC_V }} - uses: fortran-lang/setup-fortran@v1 - with: - compiler: ${{ env.FC }} - version: ${{ env.FC_V }} - - - name: Setup Graphviz - if: runner.os == 'Linux' - uses: ts-graphviz/setup-graphviz@v2 - - - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.19.1 - manifest-path: "pixi.toml" - - - name: pixi post-install - working-directory: pymake - run: | - pixi run postinstall - - - name: Download examples for pytest runs - run: | - pixi run download-examples - - - name: test on Linux - if: runner.os == 'Linux' - run: | - pixi run autotest - - - name: test on MacOS - if: runner.os == 'macOS' - run: | - pixi run autotest-base - - - name: test on Windows - if: runner.os == 'Windows' - shell: cmd - run: | - pixi run autotest-Windows - - - name: Print coverage report before upload - run: | - pixi run coverage-report - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./autotest/coverage.xml diff --git a/autotest/test_build.py b/autotest/test_build.py index 85a8b5df..11a4e454 100644 --- a/autotest/test_build.py +++ b/autotest/test_build.py @@ -5,11 +5,11 @@ import pytest from flaky import flaky -from modflow_devtools.misc import get_ostag, set_dir, set_env +from modflow_devtools.misc import get_ostag, set_dir import pymake -RERUNS = 3 +RERUNS = 1 targets = pymake.usgs_program_data.get_keys(current=True) test_ostag = get_ostag() @@ -22,45 +22,31 @@ meson_exclude = ("sutra",) targets_meson = [t for t in targets if t not in meson_exclude] -if "win" in test_ostag and test_fc_env in ("ifort",): - targets_make = [] -else: - targets_make = [ - t - for t in targets - if t not in ("libmf6", "gridgen", "mf2000", "swtv4", "mflgr") - ] +make_exclude = ("libmf6", "gridgen", "mf2000", "swtv4", "mflgr") +targets_make = [t for t in targets if t not in make_exclude] -def build_with_makefile(target, path, fc): +def build_with_makefile(target): success = True - with set_dir(path): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {target} with makefile") - os.system("make clean") - - # build MODFLOW-NWT with makefile - print(f"build {target} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - errmsg = f"{target} created by makefile does not exist." - if sys.platform.lower() == "win32" and fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that MODFLOW-NWT was made - else: - success = os.path.isfile(target) + if os.path.isfile("makefile"): + # wait to delete on windows + if sys.platform.lower() == "win32": + time.sleep(6) + + # clean prior to make + print(f"clean {target} with makefile") + os.system("make clean") + + print(f"build {target} with makefile") + return_code = os.system("make") + + success = os.path.isfile(target) + if success: + errmsg = "" else: - errmsg = "makefile does not exist" + errmsg = f"{target} created by makefile does not exist." + else: + errmsg = "makefile does not exist" return success, errmsg @@ -89,11 +75,10 @@ def test_build(function_tmpdir, target: str) -> None: @flaky(max_runs=RERUNS) @pytest.mark.parametrize("target", targets_meson) def test_meson_build(function_tmpdir, target: str) -> None: - kwargs = {} - cc = os.environ.get("CC", "gcc") fc = os.environ.get("FC", "gfortran") + cc = os.environ.get("CC", "gcc") pymake.linker_update_environment(cc=cc, fc=fc) - with set_dir(function_tmpdir), set_env(**kwargs): + with set_dir(function_tmpdir): assert ( pymake.build_apps( target, @@ -107,20 +92,21 @@ def test_meson_build(function_tmpdir, target: str) -> None: @pytest.mark.base @flaky(max_runs=RERUNS) +@pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize("target", targets_make) def test_makefile_build(function_tmpdir, target: str) -> None: - pm = pymake.Pymake(verbose=True) - pm.target = target - pm.makefile = True - pm.makefiledir = "." - pm.inplace = True - pm.dryrun = True - pm.makeclean = False - with set_dir(function_tmpdir): + pm = pymake.Pymake(verbose=True) + pm.target = target + pm.makefile = True + pm.makefiledir = "." + pm.inplace = True + pm.dryrun = True + pm.makeclean = False + pm.download_target(target) assert pm.download, f"could not download {target} distribution" assert pm.build() == 0, f"could not compile {target}" - success, errmsg = build_with_makefile(target, function_tmpdir, pm.fc) - assert success, errmsg + success, errmsg = build_with_makefile(target) + assert success, errmsg diff --git a/autotest/test_gridgen.py b/autotest/test_gridgen.py index 8f6194b5..00a0aece 100644 --- a/autotest/test_gridgen.py +++ b/autotest/test_gridgen.py @@ -7,10 +7,12 @@ import pymake +TARGET_NAME = "gridgen" + @pytest.fixture(scope="module") def target(module_tmpdir) -> pl.Path: - name = "gridgen" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -59,6 +61,7 @@ def run_gridgen(cmd, ws, exe): @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) @@ -66,12 +69,14 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "cmd", diff --git a/autotest/test_mf2005.py b/autotest/test_mf2005.py index 51887fa1..fc7bb6b1 100644 --- a/autotest/test_mf2005.py +++ b/autotest/test_mf2005.py @@ -6,10 +6,12 @@ import pymake +TARGET_NAME = "mf2005" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - target = "mf2005" + target = TARGET_NAME if sys.platform.lower() == "win32": target += ".exe" return module_tmpdir / target @@ -43,6 +45,7 @@ def run_mf2005(namefile, ws, exe): @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) @@ -50,12 +53,14 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_mf6.py b/autotest/test_mf6.py index 3ae73c1c..2f2e9619 100644 --- a/autotest/test_mf6.py +++ b/autotest/test_mf6.py @@ -10,6 +10,8 @@ import pymake +TARGET_NAME = "mf6" + # set fpth based on current path if os.path.basename(os.path.normpath(os.getcwd())) == "autotest": fpth = Path("temp") @@ -26,7 +28,7 @@ @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - name = "mf6" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -68,38 +70,8 @@ def pm(module_tmpdir, target) -> pymake.Pymake: pm.finalize() -def build_with_makefile(pm, workspace, exe): - exe_path = Path(exe) - success = False - with set_dir(workspace): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {exe} with makefile") - os.system("make clean") - - # build MODFLOW 6 with makefile - print(f"build {exe} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - if sys.platform.lower() == "win32" and pm.fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that target was made - else: - success = exe_path.is_file() - - return success - - @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) @@ -107,12 +79,14 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize("ws", sim_dirs) def test_mf6(ws, target): diff --git a/autotest/test_mf6_existing_meson.py b/autotest/test_mf6_existing_meson.py index 26cd762b..4413c80d 100644 --- a/autotest/test_mf6_existing_meson.py +++ b/autotest/test_mf6_existing_meson.py @@ -5,15 +5,18 @@ from typing import List import pytest +from modflow_devtools.misc import set_dir from modflow_devtools.ostags import get_binary_suffixes import pymake from pymake import linker_update_environment +TARGET_NAME = "mf6" + @pytest.fixture(scope="module") def targets() -> List[Path]: - target = "mf6" + target = TARGET_NAME ext, shared_ext = get_binary_suffixes() executables = [target, "zbud6", "mf5to6", "libmf6"] for idx, _ in enumerate(executables[:3]): @@ -47,54 +50,57 @@ def pm(workspace, targets) -> pymake.Pymake: @pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) def test_build_with_existing_meson(pm, module_tmpdir, workspace, targets): - # set default compilers - fc, cc = "gfortran", "gcc" - - # get the arguments - for idx, arg in enumerate(sys.argv): - if arg == "-fc": - fc = sys.argv[idx + 1] - elif "-fc=" in arg: - fc = arg.split("=")[1] - if arg == "-cc": - cc = sys.argv[idx + 1] - elif "-cc=" in arg: - cc = arg.split("=")[1] - - # check if fc differs from environmental variable - fc_env = os.environ.get("FC") - if fc_env is not None: - if fc != fc_env: - fc = fc_env - - # check if cc differs from environmental variable - cc_env = os.environ.get("CC") - if cc_env is not None: - if cc != cc_env: - cc = cc_env - - # print fortran and c/c++ compilers - print(f"fortran compiler={fc}\n" + f"c/c++ compiler={cc}\n") - - # download modflow 6 - pm.download_target(targets[0], download_path=module_tmpdir) - assert pm.download, f"could not download {targets[0]} distribution" - - linker_update_environment(cc=cc, fc=fc) - - # make modflow 6 with existing meson.build file - returncode = pymake.meson_build( - workspace, - fc, - cc, - appdir=pm.appdir, - ) - assert ( - returncode == 0 - ), "could not build modflow 6 applications using existing meson.build file" - - # check that all of the executables exist - for executable in targets: - exe_pth = os.path.join(pm.appdir, executable) - assert os.path.isfile(exe_pth), f"{exe_pth} does not exist" + with set_dir(module_tmpdir): + # set default compilers + fc, cc = "gfortran", "gcc" + + # get the arguments + for idx, arg in enumerate(sys.argv): + if arg == "-fc": + fc = sys.argv[idx + 1] + elif "-fc=" in arg: + fc = arg.split("=")[1] + if arg == "-cc": + cc = sys.argv[idx + 1] + elif "-cc=" in arg: + cc = arg.split("=")[1] + + # check if fc differs from environmental variable + fc_env = os.environ.get("FC") + if fc_env is not None: + if fc != fc_env: + fc = fc_env + + # check if cc differs from environmental variable + cc_env = os.environ.get("CC") + if cc_env is not None: + if cc != cc_env: + cc = cc_env + + # print fortran and c/c++ compilers + print(f"fortran compiler={fc}\n" + f"c/c++ compiler={cc}\n") + + # download modflow 6 + pm.download_target(targets[0], download_path=module_tmpdir) + assert pm.download, f"could not download {targets[0]} distribution" + + linker_update_environment(cc=cc, fc=fc) + + # make modflow 6 with existing meson.build file + returncode = pymake.meson_build( + workspace, + fc, + cc, + appdir=pm.appdir, + ) + assert returncode == 0, ( + "could not build modflow 6 applications " + + "using existing meson.build file" + ) + + # check that all of the executables exist + for executable in targets: + exe_pth = os.path.join(pm.appdir, executable) + assert os.path.isfile(exe_pth), f"{exe_pth} does not exist" diff --git a/autotest/test_mfusg.py b/autotest/test_mfusg.py index ab8c3d92..22fc17c0 100644 --- a/autotest/test_mfusg.py +++ b/autotest/test_mfusg.py @@ -68,6 +68,7 @@ def run_mfusg(fn, exe): @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group("mfusg") @pytest.mark.regression def test_download(pm, pm_gsi, module_tmpdir, targets): pm.download_target(targets[0], download_path=module_tmpdir) @@ -78,6 +79,7 @@ def test_download(pm, pm_gsi, module_tmpdir, targets): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group("mfusg") @pytest.mark.regression def test_compile(pm, pm_gsi, targets): assert pm.build() == 0, f"could not compile {targets[0]}" @@ -88,6 +90,7 @@ def test_compile(pm, pm_gsi, targets): @pytest.mark.dependency(name="test", depends=["download", "build"]) +@pytest.mark.xdist_group("mfusg") @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_mp6.py b/autotest/test_mp6.py index 70dfc2b9..abea072d 100644 --- a/autotest/test_mp6.py +++ b/autotest/test_mp6.py @@ -8,10 +8,12 @@ import pymake +TARGET_NAME = "mp6" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - name = "mp6" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -60,6 +62,7 @@ def update_files(fn, workspace): @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) @@ -67,12 +70,14 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", [f"EXAMPLE-{n}.mpsim" for n in range(1, 10)] diff --git a/autotest/test_mp7.py b/autotest/test_mp7.py index 5b4265fb..80b6fa28 100644 --- a/autotest/test_mp7.py +++ b/autotest/test_mp7.py @@ -10,10 +10,12 @@ ext = ".exe" if system() == "Windows" else "" +TARGET_NAME = "mp7" + @pytest.fixture(scope="module") def target(module_tmpdir): - name = "mp7" + name = TARGET_NAME return module_tmpdir / f"{name}{ext}" @@ -143,6 +145,7 @@ def run_modpath7(namefile, mp7_exe, mf2005_exe, mfusg_exe, mf6_exe): @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) @@ -150,12 +153,14 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="download_exes") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download_exes(module_tmpdir): pymake.getmfexes( @@ -166,6 +171,7 @@ def test_download_exes(module_tmpdir): @pytest.mark.dependency( name="test", depends=["download", "download_exes", "build"] ) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_mt3d.py b/autotest/test_mt3d.py index fd09c89f..f7f8e272 100644 --- a/autotest/test_mt3d.py +++ b/autotest/test_mt3d.py @@ -7,10 +7,14 @@ import pymake +TARGET_NAME = "mt3dusgs" +EXT = ".exe" if system() == "Windows" else "" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - return module_tmpdir / "mt3dusgs" + name = TARGET_NAME + return module_tmpdir / f"{TARGET_NAME}{EXT}" @pytest.fixture(scope="module") @@ -88,6 +92,7 @@ def run_mt3dusgs(workspace, mt3dms_exe, mfnwt_exe, mf6_exe): @pytest.mark.dependency(name="download_mt3dms") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download_mt3dms(pm, module_tmpdir): pm.target = "mt3dms" @@ -96,12 +101,14 @@ def test_download_mt3dms(pm, module_tmpdir): @pytest.mark.dependency(name="build_mt3dms", depends=["download_mt3dms"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile_mt3dms(pm): assert pm.build() == 0, f"could not compile {pm.target}" @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.reset(str(target)) @@ -110,6 +117,7 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @@ -117,11 +125,13 @@ def test_compile(pm, target): @pytest.mark.regression @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +@pytest.mark.xdist_group(TARGET_NAME) def test_download_exes(module_tmpdir): pymake.getmfexes(module_tmpdir, exes=("mfnwt", "mf6"), verbose=True) @pytest.mark.regression +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") @pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize( diff --git a/autotest/test_seawat.py b/autotest/test_seawat.py index d64fc13c..d20183f1 100644 --- a/autotest/test_seawat.py +++ b/autotest/test_seawat.py @@ -9,10 +9,12 @@ import pymake +TARGET_NAME = "swtv4" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - name = "swtv4" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -76,6 +78,7 @@ def build_seawat_dependency_graphs(src_path, dep_path): @pytest.mark.dependency(name="download") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) @@ -83,12 +86,14 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", @@ -122,6 +127,7 @@ def test_seawat(namefile, workspace, target): @pytest.mark.dependency(name="graph", depends=["test"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_dependency_graphs(workspace, prog_data): src_path = workspace / prog_data.srcdir diff --git a/pixi.toml b/pixi.toml index 6a802543..bc9afd61 100644 --- a/pixi.toml +++ b/pixi.toml @@ -42,9 +42,8 @@ test = "meson test --verbose --no-rebuild -C" # test download-examples = {cmd = "python ci_setup.py", cwd = "autotest"} -autotest = { cmd = "pytest -v -n auto --dist=loadfile -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } -autotest-base = { cmd = "pytest -v -n auto --dist=loadfile -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } -autotest-Windows = { cmd = "pytest -v -m='base' --durations 0 --cov=pymake --cov-report=xml --basetemp=$RUNNER_TEMP/pytest_temp --keep-failed .failed", cwd = "autotest" } +autotest = { cmd = "pytest -v -n auto --dist=loadgroup -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-base = { cmd = "pytest -v -n auto --dist=loadgroup -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } autotest-request = { cmd = "pytest -v -n=auto -m='requests' --durations=0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } autotest-schedule = { cmd = "pytest -v -m='schedule' --durations=0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } From f0c96f0b4b1820ae77c28b615e30102a35e8cdb6 Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Sun, 16 Jun 2024 16:06:02 -0500 Subject: [PATCH 5/6] * update --- autotest/test_mt3d.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/autotest/test_mt3d.py b/autotest/test_mt3d.py index f7f8e272..b3f0478a 100644 --- a/autotest/test_mt3d.py +++ b/autotest/test_mt3d.py @@ -1,6 +1,7 @@ import os import sys from pathlib import Path +from platform import system import flopy import pytest @@ -124,7 +125,8 @@ def test_compile(pm, target): @pytest.mark.regression -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +# @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +@pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.xdist_group(TARGET_NAME) def test_download_exes(module_tmpdir): pymake.getmfexes(module_tmpdir, exes=("mfnwt", "mf6"), verbose=True) @@ -132,7 +134,7 @@ def test_download_exes(module_tmpdir): @pytest.mark.regression @pytest.mark.xdist_group(TARGET_NAME) -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +# @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") @pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize( "ws", From 83fc9890e9b86159dc93ac58c25cf0944514df7c Mon Sep 17 00:00:00 2001 From: Joseph D Hughes Date: Sun, 16 Jun 2024 16:42:32 -0500 Subject: [PATCH 6/6] * update --- autotest/{test_mt3d.py => test_mt3dusgs.py} | 16 ---------------- 1 file changed, 16 deletions(-) rename autotest/{test_mt3d.py => test_mt3dusgs.py} (88%) diff --git a/autotest/test_mt3d.py b/autotest/test_mt3dusgs.py similarity index 88% rename from autotest/test_mt3d.py rename to autotest/test_mt3dusgs.py index b3f0478a..eb589ba5 100644 --- a/autotest/test_mt3d.py +++ b/autotest/test_mt3dusgs.py @@ -92,22 +92,6 @@ def run_mt3dusgs(workspace, mt3dms_exe, mfnwt_exe, mf6_exe): return success -@pytest.mark.dependency(name="download_mt3dms") -@pytest.mark.xdist_group(TARGET_NAME) -@pytest.mark.regression -def test_download_mt3dms(pm, module_tmpdir): - pm.target = "mt3dms" - pm.download_target(pm.target, download_path=module_tmpdir) - assert pm.download, f"could not download {pm.target} distribution" - - -@pytest.mark.dependency(name="build_mt3dms", depends=["download_mt3dms"]) -@pytest.mark.xdist_group(TARGET_NAME) -@pytest.mark.regression -def test_compile_mt3dms(pm): - assert pm.build() == 0, f"could not compile {pm.target}" - - @pytest.mark.dependency(name="download") @pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression