diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..f9935c63 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,192 @@ +name: pymake continuous integration + +on: + schedule: + - cron: '0 7 * * *' # run at 7 AM UTC every day + push: + paths-ignore: + - 'README.md' + - 'docs/*.md' + pull_request: + branches: + - master + - develop + paths-ignore: + - 'README.md' + - 'docs/*.md' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + + pymake-os-compiler: + name: pymake CI on different OSs with gcc and intel-classic + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # test latest gcc and python + - {os: ubuntu-latest, FC: gcc, FC_V: 13} + - {os: macos-latest, FC: gcc, FC_V: 13} + - {os: windows-latest, FC: gcc, FC_V: 13} + # test latest python and intel-classic + - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} + - {os: macos-13, FC: intel-classic, FC_V: 2021.7} + - {os: windows-2019, FC: intel-classic, FC_V: 2021.7} + # test latest python and previous gcc + - {os: ubuntu-latest, FC: gcc, FC_V: 12} + - {os: ubuntu-latest, FC: gcc, FC_V: 11} + defaults: + run: + shell: bash + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - uses: maxim-lobanov/setup-xcode@v1 + if: ${{ (runner.os == 'macOS') && (matrix.FC == 'intel-classic') }} + with: + xcode-version: "14.3.1" + + - name: Setup Graphviz on Linux + if: runner.os == 'Linux' + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.19.1 + manifest-path: "pixi.toml" + + - name: pixi post-install + working-directory: pymake + run: | + pixi run postinstall + + - name: Setup ${{ matrix.FC }} ${{ matrix.FC_V }} on ${{ matrix.os }} + uses: fortran-lang/setup-fortran@main + with: + compiler: ${{ matrix.FC }} + version: ${{ matrix.FC_V }} + + - name: Download examples for pytest runs + run: | + pixi run download-examples + + - name: Install make + if: runner.os == 'Windows' + run: choco install make + + - name: test on Linux + if: runner.os == 'Linux' + run: | + pixi run autotest + + - name: test on MacOS + if: runner.os == 'macOS' + run: | + pixi run autotest-base + + - name: test on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + pixi run autotest-base + + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} + path: ./autotest/.failed + + - name: Print coverage report before upload + run: | + pixi run coverage-report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + file: ./autotest/coverage.xml + + pymake-schedule: + name: pymake scheduled CI different OSs with gcc and intel-classic + if: ${{ github.event_name == 'schedule' }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # test latest gcc and python + - {os: ubuntu-latest, FC: gcc, FC_V: 13} + - {os: macos-latest, FC: gcc, FC_V: 13} + - {os: windows-latest, FC: gcc, FC_V: 13} + # test latest python and intel + - {os: ubuntu-latest, FC: intel-classic, FC_V: 2021.7} + - {os: macos-13, FC: intel-classic, FC_V: 2021.7} + - {os: windows-2019, FC: intel-classic, FC_V: 2021.7} + defaults: + run: + shell: bash + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - uses: maxim-lobanov/setup-xcode@v1 + if: ${{ (runner.os == 'macOS') && (matrix.FC == 'intel-classic') }} + with: + xcode-version: "14.3.1" + + - name: Setup Graphviz on Linux + if: runner.os == 'Linux' + uses: ts-graphviz/setup-graphviz@v2 + + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + pixi-version: v0.19.1 + manifest-path: "pixi.toml" + + - name: pixi post-install + working-directory: pymake + run: | + pixi run postinstall + + - name: Setup ${{ matrix.FC }} ${{ matrix.FC_V }} on ${{ matrix.os }} + uses: fortran-lang/setup-fortran@main + with: + compiler: ${{ matrix.FC }} + version: ${{ matrix.FC_V }} + + - name: Install make + if: runner.os == 'Windows' + run: choco install make + + - name: Run scheduled tests + run: | + pixi run autotest-schedule + + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-schedule-${{ matrix.os }}-${{ matrix.FC }}-${{ matrix.FC_V }} + path: ./autotest/.failed + + - name: Print coverage report before upload + run: | + pixi run coverage-report + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + file: ./autotest/coverage.xml + + diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 8c901ed5..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: pymake continuous integration - -on: - schedule: - - cron: '0 7 * * *' # run at 7 AM UTC every day - push: - paths-ignore: - - 'README.md' - - 'docs/*.md' - pull_request: - branches: - - master - - develop - paths-ignore: - - 'README.md' - - 'docs/*.md' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -jobs: - - pymakeCI-os-intel: - name: pymake CI intel on different OSs - runs-on: ${{ matrix.os }} - env: - FC: intel-classic - FC_V: "2021.7" - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-13, windows-2019] - defaults: - run: - shell: bash - - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - uses: maxim-lobanov/setup-xcode@v1 - if: runner.os == 'macOS' - with: - xcode-version: "14.3.1" - - - name: Setup ${{ env.FC }} ${{ env.FC_V }} - uses: fortran-lang/setup-fortran@v1 - with: - compiler: ${{ env.FC }} - version: ${{ env.FC_V }} - - - name: Setup Graphviz - if: runner.os == 'Linux' - uses: ts-graphviz/setup-graphviz@v2 - - - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.19.1 - manifest-path: "pixi.toml" - - - name: pixi post-install - working-directory: pymake - run: | - pixi run postinstall - - - name: Download examples for pytest runs - run: | - pixi run download-examples - - - name: test on Linux - if: runner.os == 'Linux' - run: | - pixi run autotest - - - name: test on MacOS - if: runner.os == 'macOS' - run: | - pixi run autotest-base - - - name: test on Windows - if: runner.os == 'Windows' - shell: cmd - run: | - pixi run autotest-Windows - - - name: Print coverage report before upload - run: | - pixi run coverage-report - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./autotest/coverage.xml diff --git a/.github/workflows/pymake-gcc.yml b/.github/workflows/pymake-gcc.yml deleted file mode 100644 index d514ba78..00000000 --- a/.github/workflows/pymake-gcc.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: pymake gcc build - -on: - schedule: - - cron: '0 7 * * *' # run at 7 AM UTC every day - push: - paths-ignore: - - 'README.md' - - 'docs/*.md' - pull_request: - branches: - - master - - develop - paths-ignore: - - 'README.md' - - 'docs/*.md' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -jobs: - - pymakeCI-os-gcc: - name: pymake CI gcc on different OSs - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - # test latest gcc and python - - {os: ubuntu-latest, gcc: 13} - - {os: windows-latest, gcc: 13} - - {os: macos-latest, gcc: 13} - # test latest python and previous gcc - - {os: ubuntu-latest, gcc: 12} - - {os: ubuntu-latest, gcc: 11} - defaults: - run: - shell: bash - - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - name: Setup Graphviz on Linux - if: runner.os == 'Linux' - uses: ts-graphviz/setup-graphviz@v2 - - - name: Setup pixi - uses: prefix-dev/setup-pixi@v0.8.1 - with: - pixi-version: v0.19.1 - manifest-path: "pixi.toml" - - - name: pixi post-install - working-directory: pymake - run: | - pixi run postinstall - - - name: Setup GNU Fortran - uses: fortran-lang/setup-fortran@main - with: - compiler: gcc - version: ${{ matrix.gcc }} - - - name: Download examples for pytest runs - run: | - pixi run download-examples - - - name: Install make - if: runner.os == 'Windows' - run: choco install make - - - name: Run pytest on Linux and MacOS - if: runner.os != 'Windows' - run: | - pixi run autotest-base - - - name: test on Windows - if: runner.os == 'Windows' - shell: pwsh - run: | - pixi run autotest-Windows - - - name: Print coverage report before upload - run: | - pixi run coverage-report - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - with: - file: ./autotest/coverage.xml diff --git a/.github/workflows/pymake-requests.yml b/.github/workflows/pymake-requests.yml index 514d0e1e..472efb90 100644 --- a/.github/workflows/pymake-requests.yml +++ b/.github/workflows/pymake-requests.yml @@ -30,32 +30,32 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Setup pixi + uses: prefix-dev/setup-pixi@v0.8.1 with: - python-version: "3.12" + pixi-version: v0.19.1 + manifest-path: "pixi.toml" - - name: Install python packages + - name: pixi post-install + working-directory: pymake run: | - python -m pip install --upgrade pip - pip install ".[test]" + pixi run postinstall - name: Run pytest - working-directory: ./autotest run: | - pytest -v -n=auto -m requests --durations=0 --cov=pymake --cov-report=xml + pixi run autotest-request - - name: Run scheduled tests - if: ${{ github.event_name == 'schedule' }} - working-directory: ./autotest - run: | - pytest -v -m="schedule" --durations=0 --cov=pymake --cov-report=xml + - name: Upload failed test output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: failed-requests + path: ./autotest/.failed - name: Print coverage report before upload - working-directory: ./autotest run: | - coverage report - + pixi run coverage-report + - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 env: diff --git a/.gitignore b/.gitignore index f700a606..66ff0ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ target/ # files in the autotest directory autotest/temp*/ +autotest/.failed/ autotest/*.json autotest/code.md mod_temp/ diff --git a/autotest/test_build.py b/autotest/test_build.py index 1e02c2e8..11a4e454 100644 --- a/autotest/test_build.py +++ b/autotest/test_build.py @@ -5,18 +5,13 @@ import pytest from flaky import flaky -from modflow_devtools.misc import get_ostag, set_dir, set_env +from modflow_devtools.misc import get_ostag, set_dir import pymake -RERUNS = 3 +RERUNS = 1 targets = pymake.usgs_program_data.get_keys(current=True) -targets_make = [ - t - for t in targets - if t not in ("libmf6", "gridgen", "mf2000", "swtv4", "mflgr") -] test_ostag = get_ostag() test_fc_env = os.environ.get("FC") if "win" in test_ostag: @@ -27,36 +22,31 @@ meson_exclude = ("sutra",) targets_meson = [t for t in targets if t not in meson_exclude] +make_exclude = ("libmf6", "gridgen", "mf2000", "swtv4", "mflgr") +targets_make = [t for t in targets if t not in make_exclude] -def build_with_makefile(target, path, fc): + +def build_with_makefile(target): success = True - with set_dir(path): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {target} with makefile") - os.system("make clean") - - # build MODFLOW-NWT with makefile - print(f"build {target} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - errmsg = f"{target} created by makefile does not exist." - if sys.platform.lower() == "win32" and fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that MODFLOW-NWT was made - else: - success = os.path.isfile(target) + if os.path.isfile("makefile"): + # wait to delete on windows + if sys.platform.lower() == "win32": + time.sleep(6) + + # clean prior to make + print(f"clean {target} with makefile") + os.system("make clean") + + print(f"build {target} with makefile") + return_code = os.system("make") + + success = os.path.isfile(target) + if success: + errmsg = "" else: - errmsg = "makefile does not exist" + errmsg = f"{target} created by makefile does not exist." + else: + errmsg = "makefile does not exist" return success, errmsg @@ -85,11 +75,10 @@ def test_build(function_tmpdir, target: str) -> None: @flaky(max_runs=RERUNS) @pytest.mark.parametrize("target", targets_meson) def test_meson_build(function_tmpdir, target: str) -> None: - kwargs = {} - cc = os.environ.get("CC", "gcc") fc = os.environ.get("FC", "gfortran") + cc = os.environ.get("CC", "gcc") pymake.linker_update_environment(cc=cc, fc=fc) - with set_dir(function_tmpdir), set_env(**kwargs): + with set_dir(function_tmpdir): assert ( pymake.build_apps( target, @@ -106,18 +95,18 @@ def test_meson_build(function_tmpdir, target: str) -> None: @pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize("target", targets_make) def test_makefile_build(function_tmpdir, target: str) -> None: - pm = pymake.Pymake(verbose=True) - pm.target = target - pm.makefile = True - pm.makefiledir = "." - pm.inplace = True - pm.dryrun = True - pm.makeclean = False - with set_dir(function_tmpdir): + pm = pymake.Pymake(verbose=True) + pm.target = target + pm.makefile = True + pm.makefiledir = "." + pm.inplace = True + pm.dryrun = True + pm.makeclean = False + pm.download_target(target) assert pm.download, f"could not download {target} distribution" assert pm.build() == 0, f"could not compile {target}" - success, errmsg = build_with_makefile(target, function_tmpdir, pm.fc) - assert success, errmsg + success, errmsg = build_with_makefile(target) + assert success, errmsg diff --git a/autotest/test_cli_cmds.py b/autotest/test_cli_cmds.py index f31e0a37..4e635899 100644 --- a/autotest/test_cli_cmds.py +++ b/autotest/test_cli_cmds.py @@ -46,43 +46,45 @@ def run_cli_cmd(cmd: list) -> None: @pytest.mark.base @pytest.mark.parametrize("target", targets) def test_make_program(function_tmpdir, target: str) -> None: - cmd = [ - "make-program", - target, - "--appdir", - str(function_tmpdir), - "--verbose", - ] - run_cli_cmd(cmd) + with set_dir(function_tmpdir): + cmd = [ + "make-program", + target, + "--appdir", + str(function_tmpdir), + "--verbose", + ] + run_cli_cmd(cmd) @flaky(max_runs=RERUNS) @pytest.mark.dependency(name="make_program") @pytest.mark.base def test_make_program_double(function_tmpdir) -> None: - cmd = [ - "make-program", - "mf2005", - "--double", - "--verbose", - "--appdir", - str(function_tmpdir), - ] - run_cli_cmd(cmd) + with set_dir(function_tmpdir): + cmd = [ + "make-program", + "mf2005", + "--double", + "--verbose", + "--appdir", + str(function_tmpdir), + ] + run_cli_cmd(cmd) @pytest.mark.dependency(name="make_program_all") @pytest.mark.schedule def test_make_program_all(module_tmpdir) -> None: - cmd = [ - "make-program", - ":", - "--appdir", - str(module_tmpdir / "all"), - "--verbose", - "--dryrun", - ] - run_cli_cmd(cmd) + with set_dir(module_tmpdir): + cmd = [ + "make-program", + ":", + "--appdir", + str(module_tmpdir / "all"), + "--verbose", + ] + run_cli_cmd(cmd) @flaky(max_runs=RERUNS) diff --git a/autotest/test_gridgen.py b/autotest/test_gridgen.py index 1c693126..00a0aece 100644 --- a/autotest/test_gridgen.py +++ b/autotest/test_gridgen.py @@ -7,10 +7,12 @@ import pymake +TARGET_NAME = "gridgen" + @pytest.fixture(scope="module") def target(module_tmpdir) -> pl.Path: - name = "gridgen" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -59,19 +61,22 @@ def run_gridgen(cmd, ws, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "cmd", diff --git a/autotest/test_mf2005.py b/autotest/test_mf2005.py index dbf361ea..fc7bb6b1 100644 --- a/autotest/test_mf2005.py +++ b/autotest/test_mf2005.py @@ -6,10 +6,12 @@ import pymake +TARGET_NAME = "mf2005" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - target = "mf2005" + target = TARGET_NAME if sys.platform.lower() == "win32": target += ".exe" return module_tmpdir / target @@ -43,19 +45,22 @@ def run_mf2005(namefile, ws, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target}" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_mf6.py b/autotest/test_mf6.py index b9470f64..2f2e9619 100644 --- a/autotest/test_mf6.py +++ b/autotest/test_mf6.py @@ -10,6 +10,8 @@ import pymake +TARGET_NAME = "mf6" + # set fpth based on current path if os.path.basename(os.path.normpath(os.getcwd())) == "autotest": fpth = Path("temp") @@ -26,7 +28,7 @@ @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - name = "mf6" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -68,89 +70,25 @@ def pm(module_tmpdir, target) -> pymake.Pymake: pm.finalize() -def build_with_makefile(pm, workspace, exe): - exe_path = Path(exe) - success = False - with set_dir(workspace): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {exe} with makefile") - os.system("make clean") - - # build MODFLOW 6 with makefile - print(f"build {exe} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - if sys.platform.lower() == "win32" and pm.fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that target was made - else: - success = exe_path.is_file() - - return success - - @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize("ws", sim_dirs) def test_mf6(ws, target): success, _ = flopy.run_model(target, None, model_ws=ws, silent=False) assert success, f"could not run {ws}" - - -@pytest.mark.dependency(name="makefile", depends=["build"]) -@pytest.mark.base -def test_makefile(pm, module_tmpdir, target): - assert build_with_makefile( - pm, module_tmpdir, target - ), f"could not compile {target} with makefile" - - -@pytest.mark.dependency(name="shared", depends=["makefile"]) -@pytest.mark.base -def test_sharedobject(pm, module_tmpdir, workspace, target_so, prog_data): - # reconfigure pymake object - pm.target = str(target_so) - pm.appdir = module_tmpdir - pm.srcdir = workspace / prog_data.srcdir - pm.srcdir2 = workspace / "src" - pm.excludefiles = [os.path.join(pm.srcdir2, "mf6.f90")] - pm.makefile = True - pm.makeclean = True - pm.sharedobject = True - pm.inplace = True - pm.dryrun = False - - # build the target - assert pm.build() == 0, f"could not compile {pm.target}" - assert target_so.is_file() - - -@pytest.mark.dependency(name="shared_makefile", depends=["shared", "makefile"]) -@pytest.mark.base -def test_sharedobject_makefile(pm, module_tmpdir, target_so): - assert build_with_makefile( - pm, module_tmpdir, target_so - ), f"could not compile {target_so} with makefile" diff --git a/autotest/test_mf6_existing_meson.py b/autotest/test_mf6_existing_meson.py index 26cd762b..4413c80d 100644 --- a/autotest/test_mf6_existing_meson.py +++ b/autotest/test_mf6_existing_meson.py @@ -5,15 +5,18 @@ from typing import List import pytest +from modflow_devtools.misc import set_dir from modflow_devtools.ostags import get_binary_suffixes import pymake from pymake import linker_update_environment +TARGET_NAME = "mf6" + @pytest.fixture(scope="module") def targets() -> List[Path]: - target = "mf6" + target = TARGET_NAME ext, shared_ext = get_binary_suffixes() executables = [target, "zbud6", "mf5to6", "libmf6"] for idx, _ in enumerate(executables[:3]): @@ -47,54 +50,57 @@ def pm(workspace, targets) -> pymake.Pymake: @pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) def test_build_with_existing_meson(pm, module_tmpdir, workspace, targets): - # set default compilers - fc, cc = "gfortran", "gcc" - - # get the arguments - for idx, arg in enumerate(sys.argv): - if arg == "-fc": - fc = sys.argv[idx + 1] - elif "-fc=" in arg: - fc = arg.split("=")[1] - if arg == "-cc": - cc = sys.argv[idx + 1] - elif "-cc=" in arg: - cc = arg.split("=")[1] - - # check if fc differs from environmental variable - fc_env = os.environ.get("FC") - if fc_env is not None: - if fc != fc_env: - fc = fc_env - - # check if cc differs from environmental variable - cc_env = os.environ.get("CC") - if cc_env is not None: - if cc != cc_env: - cc = cc_env - - # print fortran and c/c++ compilers - print(f"fortran compiler={fc}\n" + f"c/c++ compiler={cc}\n") - - # download modflow 6 - pm.download_target(targets[0], download_path=module_tmpdir) - assert pm.download, f"could not download {targets[0]} distribution" - - linker_update_environment(cc=cc, fc=fc) - - # make modflow 6 with existing meson.build file - returncode = pymake.meson_build( - workspace, - fc, - cc, - appdir=pm.appdir, - ) - assert ( - returncode == 0 - ), "could not build modflow 6 applications using existing meson.build file" - - # check that all of the executables exist - for executable in targets: - exe_pth = os.path.join(pm.appdir, executable) - assert os.path.isfile(exe_pth), f"{exe_pth} does not exist" + with set_dir(module_tmpdir): + # set default compilers + fc, cc = "gfortran", "gcc" + + # get the arguments + for idx, arg in enumerate(sys.argv): + if arg == "-fc": + fc = sys.argv[idx + 1] + elif "-fc=" in arg: + fc = arg.split("=")[1] + if arg == "-cc": + cc = sys.argv[idx + 1] + elif "-cc=" in arg: + cc = arg.split("=")[1] + + # check if fc differs from environmental variable + fc_env = os.environ.get("FC") + if fc_env is not None: + if fc != fc_env: + fc = fc_env + + # check if cc differs from environmental variable + cc_env = os.environ.get("CC") + if cc_env is not None: + if cc != cc_env: + cc = cc_env + + # print fortran and c/c++ compilers + print(f"fortran compiler={fc}\n" + f"c/c++ compiler={cc}\n") + + # download modflow 6 + pm.download_target(targets[0], download_path=module_tmpdir) + assert pm.download, f"could not download {targets[0]} distribution" + + linker_update_environment(cc=cc, fc=fc) + + # make modflow 6 with existing meson.build file + returncode = pymake.meson_build( + workspace, + fc, + cc, + appdir=pm.appdir, + ) + assert returncode == 0, ( + "could not build modflow 6 applications " + + "using existing meson.build file" + ) + + # check that all of the executables exist + for executable in targets: + exe_pth = os.path.join(pm.appdir, executable) + assert os.path.isfile(exe_pth), f"{exe_pth} does not exist" diff --git a/autotest/test_mflgr.py b/autotest/test_mflgr.py deleted file mode 100644 index 1de8c968..00000000 --- a/autotest/test_mflgr.py +++ /dev/null @@ -1,34 +0,0 @@ -from pathlib import Path - -import pytest - -import pymake - - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> Path: - target = "mflgr" - return module_tmpdir / target - - -@pytest.fixture(scope="module") -def prog_dict(target) -> dict: - return pymake.usgs_program_data.get_target(target) - - -@pytest.fixture(scope="module") -def workspace(module_tmpdir, prog_dict) -> Path: - return module_tmpdir / prog_dict.dirname - - -def compile_code(ws, exe): - return pymake.build_apps( - str(exe), download_dir=ws, appdir=ws, verbose=True - ) - - -@pytest.mark.base -def test_compile(module_tmpdir, target): - assert ( - compile_code(module_tmpdir, target) == 0 - ), f"could not compile {target}" diff --git a/autotest/test_mfnwt.py b/autotest/test_mfnwt.py deleted file mode 100644 index f7cb8013..00000000 --- a/autotest/test_mfnwt.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -import sys -import time -from pathlib import Path - -import pytest -from modflow_devtools.misc import set_dir - -import pymake - - -@pytest.fixture(scope="module") -def target(module_tmpdir) -> str: - target = "mfnwt" - return module_tmpdir / target - - -@pytest.fixture(scope="module") -def prog_data(target) -> dict: - return pymake.usgs_program_data.get_target(target.name) - - -@pytest.fixture(scope="module") -def workspace(module_tmpdir, prog_data) -> Path: - return module_tmpdir / prog_data.dirname - - -@pytest.fixture(scope="module") -def pm(module_tmpdir, target) -> pymake.Pymake: - pm = pymake.Pymake(verbose=True) - pm.target = str(target) - pm.appdir = str(module_tmpdir) - pm.makefile = True - pm.makefiledir = str(module_tmpdir) - pm.inplace = True - pm.dryrun = False - pm.verbose = True - yield pm - pm.finalize() - - -def build_with_makefile(ws): - success = True - with set_dir(ws): - if os.path.isfile("makefile"): - # wait to delete on windows - if sys.platform.lower() == "win32": - time.sleep(6) - - # clean prior to make - print(f"clean {target} with makefile") - os.system("make clean") - - # build MODFLOW-NWT with makefile - print(f"build {target} with makefile") - return_code = os.system("make") - - # test if running on Windows with ifort, if True the makefile - # should fail - errmsg = f"{target} created by makefile does not exist." - if sys.platform.lower() == "win32" and pm.fc == "ifort": - if return_code != 0: - success = True - else: - success = False - # verify that MODFLOW-NWT was made - else: - success = os.path.isfile(target) - else: - errmsg = "makefile does not exist" - - assert success, errmsg - - -@pytest.mark.dependency(name="download") -@pytest.mark.base -def test_download(pm, module_tmpdir, target): - pm.download_target(target, download_path=module_tmpdir) - assert pm.download, f"could not download {target} distribution" - - -@pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base -def test_compile(pm, target): - assert pm.build() == 0, f"could not compile {target}" - - -@pytest.mark.dependency(name="makefile", depends=["build"]) -@pytest.mark.base -def test_makefile(workspace): - build_with_makefile(workspace) diff --git a/autotest/test_mfusg.py b/autotest/test_mfusg.py index 0f13c3e3..22fc17c0 100644 --- a/autotest/test_mfusg.py +++ b/autotest/test_mfusg.py @@ -68,7 +68,8 @@ def run_mfusg(fn, exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group("mfusg") +@pytest.mark.regression def test_download(pm, pm_gsi, module_tmpdir, targets): pm.download_target(targets[0], download_path=module_tmpdir) assert pm.download, f"could not download {targets[0]}" @@ -78,7 +79,8 @@ def test_download(pm, pm_gsi, module_tmpdir, targets): @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group("mfusg") +@pytest.mark.regression def test_compile(pm, pm_gsi, targets): assert pm.build() == 0, f"could not compile {targets[0]}" assert (targets[0]).is_file() @@ -87,7 +89,8 @@ def test_compile(pm, pm_gsi, targets): assert targets[1].is_file() -@pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.dependency(name="test", depends=["download", "build"]) +@pytest.mark.xdist_group("mfusg") @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_misc_programs.py b/autotest/test_misc_programs.py deleted file mode 100644 index 93d4e30a..00000000 --- a/autotest/test_misc_programs.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -import pymake - -targets = [ - "crt", - "vs2dt", - "zonbud3", -] - - -@pytest.mark.base -@pytest.mark.parametrize("target", targets) -def test_compile(module_tmpdir, target): - bin_dir = module_tmpdir / "bin" - assert ( - pymake.build_apps( - str(bin_dir / target), - download_dir=str(module_tmpdir), - appdir=str(bin_dir), - verbose=True, - ) - == 0 - ), f"could not compile {target}" diff --git a/autotest/test_mp6.py b/autotest/test_mp6.py index 51c14522..abea072d 100644 --- a/autotest/test_mp6.py +++ b/autotest/test_mp6.py @@ -8,10 +8,12 @@ import pymake +TARGET_NAME = "mp6" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - name = "mp6" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -60,19 +62,22 @@ def update_files(fn, workspace): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", [f"EXAMPLE-{n}.mpsim" for n in range(1, 10)] diff --git a/autotest/test_mp7.py b/autotest/test_mp7.py index 24b1c669..80b6fa28 100644 --- a/autotest/test_mp7.py +++ b/autotest/test_mp7.py @@ -10,10 +10,12 @@ ext = ".exe" if system() == "Windows" else "" +TARGET_NAME = "mp7" + @pytest.fixture(scope="module") def target(module_tmpdir): - name = "mp7" + name = TARGET_NAME return module_tmpdir / f"{name}{ext}" @@ -143,19 +145,22 @@ def run_modpath7(namefile, mp7_exe, mf2005_exe, mfusg_exe, mf6_exe): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target} distribution" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="download_exes") +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_download_exes(module_tmpdir): pymake.getmfexes( @@ -166,6 +171,7 @@ def test_download_exes(module_tmpdir): @pytest.mark.dependency( name="test", depends=["download", "download_exes", "build"] ) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", diff --git a/autotest/test_mt3d.py b/autotest/test_mt3dusgs.py similarity index 84% rename from autotest/test_mt3d.py rename to autotest/test_mt3dusgs.py index f9c2ea0b..eb589ba5 100644 --- a/autotest/test_mt3d.py +++ b/autotest/test_mt3dusgs.py @@ -1,16 +1,21 @@ import os import sys from pathlib import Path +from platform import system import flopy import pytest import pymake +TARGET_NAME = "mt3dusgs" +EXT = ".exe" if system() == "Windows" else "" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - return module_tmpdir / "mt3dusgs" + name = TARGET_NAME + return module_tmpdir / f"{TARGET_NAME}{EXT}" @pytest.fixture(scope="module") @@ -87,22 +92,9 @@ def run_mt3dusgs(workspace, mt3dms_exe, mfnwt_exe, mf6_exe): return success -@pytest.mark.dependency(name="download_mt3dms") -@pytest.mark.base -def test_download_mt3dms(pm, module_tmpdir): - pm.target = "mt3dms" - pm.download_target(pm.target, download_path=module_tmpdir) - assert pm.download, f"could not download {pm.target} distribution" - - -@pytest.mark.dependency(name="build_mt3dms", depends=["download_mt3dms"]) -@pytest.mark.base -def test_compile_mt3dms(pm): - assert pm.build() == 0, f"could not compile {pm.target}" - - @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.reset(str(target)) pm.download_target(target, download_path=module_tmpdir) @@ -110,19 +102,23 @@ def test_download(pm, module_tmpdir, target): @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.regression -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +# @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +@pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") +@pytest.mark.xdist_group(TARGET_NAME) def test_download_exes(module_tmpdir): pymake.getmfexes(module_tmpdir, exes=("mfnwt", "mf6"), verbose=True) @pytest.mark.regression -@pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") +@pytest.mark.xdist_group(TARGET_NAME) +# @pytest.mark.skipif(sys.platform == "darwin", reason="do not run on OSX") @pytest.mark.skipif(sys.platform == "win32", reason="do not run on Windows") @pytest.mark.parametrize( "ws", diff --git a/autotest/test_seawat.py b/autotest/test_seawat.py index 1dfbd431..d20183f1 100644 --- a/autotest/test_seawat.py +++ b/autotest/test_seawat.py @@ -9,10 +9,12 @@ import pymake +TARGET_NAME = "swtv4" + @pytest.fixture(scope="module") def target(module_tmpdir) -> Path: - name = "swtv4" + name = TARGET_NAME ext = ".exe" if system() == "Windows" else "" return module_tmpdir / f"{name}{ext}" @@ -76,19 +78,22 @@ def build_seawat_dependency_graphs(src_path, dep_path): @pytest.mark.dependency(name="download") -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_download(pm, module_tmpdir, target): pm.download_target(target, download_path=module_tmpdir) assert pm.download, f"could not download {target}" @pytest.mark.dependency(name="build", depends=["download"]) -@pytest.mark.base +@pytest.mark.xdist_group(TARGET_NAME) +@pytest.mark.regression def test_compile(pm, target): assert pm.build() == 0, f"could not compile {target}" @pytest.mark.dependency(name="test", depends=["build"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression @pytest.mark.parametrize( "namefile", @@ -122,6 +127,7 @@ def test_seawat(namefile, workspace, target): @pytest.mark.dependency(name="graph", depends=["test"]) +@pytest.mark.xdist_group(TARGET_NAME) @pytest.mark.regression def test_dependency_graphs(workspace, prog_data): src_path = workspace / prog_data.srcdir diff --git a/autotest/test_triangle_makefile.py b/autotest/test_triangle_makefile.py deleted file mode 100644 index 45579d87..00000000 --- a/autotest/test_triangle_makefile.py +++ /dev/null @@ -1,69 +0,0 @@ -import os -import sys - -import flopy -import pytest - -import pymake - - -@pytest.mark.base -def test_pymake_makefile(module_tmpdir): - target = "triangle" - pm = pymake.Pymake(verbose=True) - pm.makefile = True - pm.makeclean = True - # pm.cc = "gcc" - - if sys.platform.lower() == "win32": - if pm.cc == "icl": - return - target += ".exe" - - # get current directory - cwd = os.getcwd() - - # change to working directory so triangle download directory is - # a subdirectory in the working directory - os.chdir(module_tmpdir) - - # build triangle and makefile - assert ( - pymake.build_apps(target, clean=False, pymake_object=pm) == 0 - ), f"could not build {target}" - - if os.path.isfile(os.path.join(module_tmpdir, "makefile")): - print("cleaning with GNU make") - # clean prior to make - print(f"clean {target} with makefile") - success, _ = flopy.run_model( - "make", - None, - cargs="clean", - model_ws=module_tmpdir, - report=True, - normal_msg="rm -rf ./triangle", - silent=False, - ) - - # build triangle with makefile - if success: - print(f"build {target} with makefile") - success, _ = flopy.run_model( - "make", - None, - model_ws=module_tmpdir, - report=True, - normal_msg="cc -O2 -o triangle ./obj_temp/triangle.o", - silent=False, - ) - - # finalize Pymake object - pm.finalize() - - # return to starting directory - os.chdir(cwd) - - assert os.path.isfile( - os.path.join(module_tmpdir, target) - ), f"could not build {target} with makefile" diff --git a/pixi.toml b/pixi.toml index 589bea5c..bc9afd61 100644 --- a/pixi.toml +++ b/pixi.toml @@ -34,18 +34,18 @@ postinstall = "pip install --no-build-isolation --no-deps --disable-pip-version- # format check-lint = "ruff check ." -fix-lint = "ruff check . --fix" check-format = "ruff format . --check" -fix-format = "ruff format ." +fix-style = "ruff check . --fix; ruff format ." # build test = "meson test --verbose --no-rebuild -C" # test download-examples = {cmd = "python ci_setup.py", cwd = "autotest"} -autotest = { cmd = "pytest -v -n auto --dist=loadfile -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } -autotest-base = { cmd = "pytest -v -n auto --dist=loadfile -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } -autotest-Windows = { cmd = "pytest -v -m='base' --durations 0 --cov=pymake --cov-report=xml --basetemp=$RUNNER_TEMP/pytest_temp --keep-failed .failed", cwd = "autotest" } +autotest = { cmd = "pytest -v -n auto --dist=loadgroup -m='base or regression' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-base = { cmd = "pytest -v -n auto --dist=loadgroup -m='base' --durations 0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-request = { cmd = "pytest -v -n=auto -m='requests' --durations=0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } +autotest-schedule = { cmd = "pytest -v -m='schedule' --durations=0 --cov=pymake --cov-report=xml --keep-failed .failed", cwd = "autotest" } # coverage report coverage-report = { cmd = "coverage report", cwd = "autotest"}