diff --git a/.github/scripts/ghcr-prune.py b/.github/scripts/ghcr-prune.py new file mode 100644 index 00000000000..87b5db7e112 --- /dev/null +++ b/.github/scripts/ghcr-prune.py @@ -0,0 +1,187 @@ +import argparse +import logging +import requests +import re +import json +from datetime import datetime +from datetime import timedelta + +description = """ +This script can be used to prune container images hosted on ghcr.io.\n + +Our testing workflow will build and push container images to ghcr.io +that are only used for testing. This script is used to cleanup these +temporary images. + +You can filter containers by any combination of name, age, and untagged. +""" + +parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawTextHelpFormatter) + +parser.add_argument("--token", required=True, help='GitHub token with "repo" scope') +parser.add_argument("--org", required=True, help="Organization name") +parser.add_argument("--name", required=True, help="Package name") +parser.add_argument( + "--age", type=int, help="Filter versions by age, removing anything older than" +) +parser.add_argument( + "--filter", help="Filter which versions are consider for pruning", default=".*" +) +parser.add_argument("--untagged", action="store_true", help="Prune untagged versions") +parser.add_argument( + "--dry-run", action="store_true", help="Does not actually delete anything" +) + +logging_group = parser.add_argument_group("logging") +logging_group.add_argument( + "--log-level", choices=("DEBUG", "INFO", "WARNING", "ERROR"), default="INFO" +) + +kwargs = vars(parser.parse_args()) + +logging.basicConfig(level=kwargs["log_level"]) + +logger = logging.getLogger("ghcr-prune") + + +class GitHubPaginate: + """Iterator for GitHub API. + + Provides small wrapper for GitHub API to utilize paging in API calls. + + https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28 + """ + def __init__(self, token, org, name, age, filter, untagged, **_): + self.token = token + self.session = None + self.url = ( + f"https://api.github.com/orgs/{org}/packages/container/{name}/versions" + ) + self.expired = datetime.now() - timedelta(days=age) + self.filter = re.compile(filter) + self.page = None + self.untagged = untagged + + def create_session(self): + self.session = requests.Session() + self.session.headers.update( + { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self.token}", + "X-GitHub-Api-Version": "2022-11-28", + } + ) + + def grab_page(self): + if self.session is None: + raise Exception("Must create session first") + + if self.url is None: + raise Exception("No more pages") + + response = self.session.get(self.url) + + response.raise_for_status() + + remaining = int(response.headers["X-RateLimit-Remaining"]) + + logger.debug(f"Remaining api limit {remaining}") + + if remaining <= 0: + reset = response.headers["X-RateLimit-Reset"] + + raise Exception(f"Hit ratelimit will reset at {reset}") + + try: + self.url = self.get_next_url(response.headers["Link"]) + except Exception as e: + logger.debug(f"No Link header found {e}") + + self.url = None + + return self.filter_results(response.json()) + + def get_next_url(self, link): + match = re.match("<([^>]*)>.*", link) + + if match is None: + raise Exception("Could not determine next link") + + return match.group(1) + + def filter_results(self, data): + results = [] + + logger.info(f"Processing {len(data)} containers") + + for x in data: + url = x["url"] + updated_at = datetime.strptime(x["updated_at"], "%Y-%m-%dT%H:%M:%SZ") + + logger.debug(f"Processing\n{json.dumps(x, indent=2)}") + + try: + tag = x["metadata"]["container"]["tags"][0] + except IndexError: + logger.info(f'Found untagged version {x["id"]}') + + if self.untagged: + results.append(url) + + continue + + if not self.filter.match(tag): + logger.info(f"Skipping {tag}, did not match filter") + + continue + + if updated_at < self.expired: + logger.info( + f"Pruning {tag}, updated at {updated_at}, expiration {self.expired}" + ) + + results.append(url) + else: + logger.info(f"Skipping {tag}, more recent than {self.expired}") + + return results + + def __iter__(self): + self.create_session() + + return self + + def __next__(self): + if self.page is None or len(self.page) == 0: + try: + self.page = self.grab_page() + except Exception as e: + logger.debug(f"StopIteration condition {e!r}") + + raise StopIteration from None + + try: + item = self.page.pop(0) + except IndexError: + raise StopIteration from None + + return item + + def remove_container(self, url): + if self.session is None: + raise Exception("Must create session first") + + response = self.session.delete(url) + + response.raise_for_status() + + logger.debug(f"{response.headers}") + + +pager = GitHubPaginate(**kwargs) + +for url in pager: + if kwargs["dry_run"]: + logger.info(f"Pruning {url}") + else: + pager.remove_container(url) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml deleted file mode 100644 index d3b0d1002dc..00000000000 --- a/.github/workflows/container.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: container build/publish - -on: - push: - branches: - - master - paths: - - 'docker/**' - - pull_request: - branches: - - master - paths: - - 'docker/**' - - workflow_dispatch: - -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -jobs: - # Only build container if there has been a change. - build-containers: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: ghcr.io/ESMCI/cime - flavor: | - latest=auto - tags: | - type=sha - - name: Build and push - uses: docker/build-push-action@v3 - with: - target: base - context: docker/ - push: ${{ github.event_name == 'push' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max diff --git a/.github/workflows/ghcr-prune.yml b/.github/workflows/ghcr-prune.yml new file mode 100644 index 00000000000..5e26f83e3ce --- /dev/null +++ b/.github/workflows/ghcr-prune.yml @@ -0,0 +1,24 @@ +name: Prune ghcr.io container images +on: + schedule: + # run once a day + - cron: '0 2 * * *' + + # Temporary to test + pull_request: + +permissions: {} + +jobs: + prune: + permissions: + packages: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - run: | + pip install requests + + # remove containers older than 14 days and only generated by testing workflow + python .github/scripts/ghcr-prune.py --token ${{ secrets.GITHUB_TOKEN }} --org esmci --name cime --age 14 --filter sha- --untagged diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ab759966c93..994a2dd1b4a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -9,6 +9,7 @@ on: - 'scripts/**' - 'tools/**' - 'utils/**' + - 'docker/**' pull_request: branches: @@ -18,6 +19,7 @@ on: - 'scripts/**' - 'tools/**' - 'utils/**' + - 'docker/**' workflow_dispatch: @@ -30,8 +32,46 @@ permissions: packages: read jobs: + build-containers: + runs-on: ubuntu-latest + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + permissions: + packages: write + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/ESMCI/cime + tags: | + type=raw,value=latest,enable=${{ github.event_name == 'push' }} + type=sha,format=long + - name: Build and push + uses: docker/build-push-action@v3 + with: + target: base + context: docker/ + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + pre-commit: runs-on: ubuntu-latest + if: ${{ github.event_name == 'pull_request' && ! cancelled() }} timeout-minutes: 2 steps: - name: Checkout code @@ -40,12 +80,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.9 - # Offical action is deprecated in favor of pre-commit.ci - # Should evaulate switching or just running manually. - # - name: Runs pre-commit action - # # Do not run if using act tooling (https://github.com/nektos/act) - # if: ${{ !env.ACT }} - # uses: pre-commit/action@v2.0.3 - name: Runs pre-commit run: | pip install pre-commit @@ -55,9 +89,10 @@ jobs: # Runs unit testing under different python versions. unit-testing: runs-on: ubuntu-latest - if: ${{ always() && ! cancelled() }} + if: ${{ github.event_name == 'pull_request' && always() && ! cancelled() }} + needs: build-containers container: - image: ghcr.io/esmci/cime:latest + image: ghcr.io/esmci/cime:${{ github.event.pull_request.head.repo.full_name == github.repository && format('sha-{0}', github.sha) || 'latest' }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -70,37 +105,37 @@ jobs: - name: Run tests shell: bash env: - INIT: "false" CIME_MODEL: "cesm" - CIME_DRIVER: "mct" - UPDATE_CIME: "true" - GIT_SHALLOW: "true" + CIME_DRIVER: "nuopc" CIME_TEST_PLATFORM: ubuntu-latest run: | - export INSTALL_PATH="${PWD}" - export CIME_REPO=https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }} - export CIME_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}} + export SRC_PATH="${GITHUB_WORKSPACE}" mamba install -y python=${{ matrix.python-version }} source /entrypoint.sh - git config --global --add safe.directory /__w/cime/cime + # GitHub runner home is different than container + cp -rf /root/.cime /github/home/ - init_cime + git status pytest -vvv --cov=CIME --machine docker --no-fortran-run CIME/tests/test_unit* # Run system tests system-testing: runs-on: ubuntu-latest - if: ${{ always() && ! cancelled() }} + if: ${{ github.event_name == 'pull_request' && always() && ! cancelled() }} + needs: build-containers container: - image: ghcr.io/esmci/cime:latest + image: ghcr.io/esmci/cime:${{ github.event.pull_request.head.repo.full_name == github.repository && format('sha-{0}', github.sha) || 'latest' }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + options: --hostname docker strategy: + # allow all jobs to finish + fail-fast: false matrix: model: ["e3sm", "cesm"] driver: ["mct", "nuopc"] @@ -108,6 +143,9 @@ jobs: # exclude nuopc driver when running e3sm tests - model: "e3sm" driver: "nuopc" + # exclude mct driver when running cesm tests + - model: "cesm" + driver: "mct" steps: - name: Checkout code uses: actions/checkout@v2 @@ -119,43 +157,48 @@ jobs: - name: Run tests shell: bash env: - INIT: "false" CIME_MODEL: ${{ matrix.model }} CIME_DRIVER: ${{ matrix.driver }} - UPDATE_CIME: "true" - GIT_SHALLOW: "true" CIME_TEST_PLATFORM: ubuntu-latest run: | - export INSTALL_PATH="${PWD}/cime" - export CIME_REPO=https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }} - export CIME_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}} + export SRC_PATH="${GITHUB_WORKSPACE}" source /entrypoint.sh - git config --global --add safe.directory /__w/cime/cime + # GitHub runner home is different than container + cp -rf /root/.cime /github/home/ - if [[ "${CIME_MODEL}" == "e3sm" ]] - then - init_e3sm - else - init_cime - fi + if [[ "${CIME_MODEL}" == "e3sm" ]]; then + git remote set-url origin https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }} + git remote set-branches origin "*" + git fetch origin + git checkout ${GITHUB_HEAD_REF:-${GITHUB_REF##*/}} + + # sync correct submodules + git submodule update - source /opt/conda/etc/profile.d/conda.sh + source /opt/conda/etc/profile.d/conda.sh - conda activate base + conda activate base + fi + + git status pytest -vvv --cov=CIME --machine docker --no-fortran-run --no-teardown CIME/tests/test_sys* + - uses: mxschmitt/action-tmate@v3 + if: ${{ !always() }} + with: + limit-access-to-actor: true - name: Create testing log archive if: ${{ failure() }} shell: bash - run: tar -czvf /testing-logs-${GITHUB_RUN_NUMBER}.tar.gz /storage/cases/ + run: tar -czvf /testing-logs-${GITHUB_RUN_NUMBER}-${{ matrix.model }}-${{ matrix.driver }}.tar.gz /storage/cases/ # How to download artifacts: # https://docs.github.com/en/actions/managing-workflow-runs/downloading-workflow-artifacts - name: Upload testing logs if: ${{ failure() }} uses: actions/upload-artifact@v3 with: - name: testing-logs-${{ github.run_number }} - path: /testing-logs-${{ github.run_number}}.tar.gz + name: testing-logs-${{ github.run_number }}-${{ matrix.model }}-${{ matrix.driver }} + path: /testing-logs-${{ github.run_number}}-${{ matrix.model }}-${{ matrix.driver }}.tar.gz retention-days: 4 diff --git a/.gitignore b/.gitignore index f6351cf8996..0f54f714f75 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ components libraries share test_coverage/** +*.bak diff --git a/.gitmodules b/.gitmodules index e69de29bb2d..13f9ecb952f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "CIME/non_py/cprnc"] + path = CIME/non_py/cprnc + url = git@github.com:ESMCI/cprnc diff --git a/CIME/BuildTools/configure.py b/CIME/BuildTools/configure.py index 4cd9059e5db..7cb884a625f 100755 --- a/CIME/BuildTools/configure.py +++ b/CIME/BuildTools/configure.py @@ -145,13 +145,13 @@ def __init__(self, compiler, mpilib, debug, comp_interface, threading=False): "DEBUG": debug, "COMP_INTERFACE": comp_interface, "PIO_VERSION": 2, - "SMP_PRESENT": threading, + "BUILD_THREADED": threading, "MODEL": get_model(), "SRCROOT": get_src_root(), } def get_build_threaded(self): - return self.get_value("SMP_PRESENT") + return self.get_value("BUILD_THREADED") def get_case_root(self): """Returns the root directory for this case.""" diff --git a/CIME/SystemTests/erp.py b/CIME/SystemTests/erp.py index f549f9e116e..6d58248c138 100644 --- a/CIME/SystemTests/erp.py +++ b/CIME/SystemTests/erp.py @@ -42,10 +42,6 @@ def _case_two_setup(self): self._case.set_value("ROOTPE_{}".format(comp), int(rootpe / 2)) RestartTest._case_two_setup(self) - self._case.case_setup(test_mode=True, reset=True) - # Note, some components, like CESM-CICE, have - # decomposition information in env_build.xml that - # needs to be regenerated for the above new tasks and thread counts def _case_one_custom_postrun_action(self): self.copy_case1_restarts_to_case2() diff --git a/CIME/SystemTests/nck.py b/CIME/SystemTests/nck.py index 5a391b5ecf7..f75a2914215 100644 --- a/CIME/SystemTests/nck.py +++ b/CIME/SystemTests/nck.py @@ -61,4 +61,3 @@ def _case_two_setup(self): if rootpe > 1: self._case.set_value("ROOTPE_{}".format(comp), int(rootpe - ntasks)) self._case.set_value("NTASKS_{}".format(comp), ntasks * 2) - self._case.case_setup(test_mode=True, reset=True) diff --git a/CIME/SystemTests/nodefail.py b/CIME/SystemTests/nodefail.py index 35f0ca3c8a6..d975cfc5bfd 100644 --- a/CIME/SystemTests/nodefail.py +++ b/CIME/SystemTests/nodefail.py @@ -24,7 +24,7 @@ def _restart_fake_phase(self): exeroot = self._case.get_value("EXEROOT") driver = self._case.get_value("COMP_INTERFACE") if driver == "nuopc": - logname = "drv" + logname = "med" else: logname = "cpl" fake_exe = """#!/bin/bash diff --git a/CIME/SystemTests/pea.py b/CIME/SystemTests/pea.py index cc9509e4e5b..4fb3a4569ca 100644 --- a/CIME/SystemTests/pea.py +++ b/CIME/SystemTests/pea.py @@ -48,4 +48,3 @@ def _case_two_setup(self): if os.path.isfile("Macros"): os.remove("Macros") - self._case.case_setup(test_mode=True, reset=True) diff --git a/CIME/SystemTests/pem.py b/CIME/SystemTests/pem.py index fc8317f432f..d98f9e4a2c7 100644 --- a/CIME/SystemTests/pem.py +++ b/CIME/SystemTests/pem.py @@ -42,4 +42,3 @@ def _case_two_setup(self): if ntasks > 1: self._case.set_value("NTASKS_{}".format(comp), int(ntasks / 2)) self._case.set_value("ROOTPE_{}".format(comp), int(rootpe / 2)) - self._case.case_setup(test_mode=True, reset=True) diff --git a/CIME/SystemTests/pet.py b/CIME/SystemTests/pet.py index 7dbaa9af79c..432d7c99303 100644 --- a/CIME/SystemTests/pet.py +++ b/CIME/SystemTests/pet.py @@ -34,12 +34,7 @@ def _case_one_setup(self): if self._case.get_value("NTHRDS_{}".format(comp)) <= 1: self._case.set_value("NTHRDS_{}".format(comp), 2) - # Need to redo case_setup because we may have changed the number of threads - def _case_two_setup(self): # Do a run with all threads set to 1 for comp in self._case.get_values("COMP_CLASSES"): self._case.set_value("NTHRDS_{}".format(comp), 1) - - # Need to redo case_setup because we may have changed the number of threads - self._case.case_setup(reset=True, test_mode=True) diff --git a/CIME/SystemTests/pgn.py b/CIME/SystemTests/pgn.py index a597cc71f97..ac68773aeb4 100644 --- a/CIME/SystemTests/pgn.py +++ b/CIME/SystemTests/pgn.py @@ -45,7 +45,9 @@ ] ) FCLD_NC = "cam.h0.cloud.nc" -INIT_COND_FILE_TEMPLATE = "20210915.v2.ne4_oQU240.F2010.{}.{}.0002-{:02d}-01-00000.nc" +INIT_COND_FILE_TEMPLATE = ( + "20231105.v3b01.F2010.ne4_oQU240.chrysalis.{}.{}.0002-{:02d}-01-00000.nc" +) INSTANCE_FILE_TEMPLATE = "{}{}_{:04d}.h0.0001-01-01-00000{}.nc" @@ -95,8 +97,8 @@ def build_phase(self, sharedlib_only=False, model_only=False): logger.debug("PGN_INFO: Updating user_nl_* files") csmdata_root = self._case.get_value("DIN_LOC_ROOT") - csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4_v2_init") - csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4_oQU240_v2_init") + csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4_v3_init") + csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4_oQU240_v3_init") iinst = 1 for icond in range(1, NUMBER_INITIAL_CONDITIONS + 1): diff --git a/CIME/SystemTests/seq.py b/CIME/SystemTests/seq.py index 304932d7d14..7413f900899 100644 --- a/CIME/SystemTests/seq.py +++ b/CIME/SystemTests/seq.py @@ -49,4 +49,3 @@ def _case_two_setup(self): rootpe += newntasks self._case.flush() - self._case.case_setup(test_mode=True, reset=True) diff --git a/CIME/SystemTests/system_tests_common.py b/CIME/SystemTests/system_tests_common.py index a9c61b28c24..236b34d0301 100644 --- a/CIME/SystemTests/system_tests_common.py +++ b/CIME/SystemTests/system_tests_common.py @@ -13,6 +13,7 @@ expect, get_current_commit, SharedArea, + is_comp_standalone, ) from CIME.test_status import * from CIME.hist_utils import ( @@ -28,7 +29,7 @@ from CIME.locked_files import LOCKED_DIR, lock_file, is_locked from CIME.baselines.performance import ( get_latest_cpl_logs, - _perf_get_memory, + perf_get_memory_list, perf_compare_memory_baseline, perf_compare_throughput_baseline, perf_write_baseline, @@ -110,7 +111,7 @@ def __init__( self._init_locked_files(caseroot, expected) self._skip_pnl = False self._cpllog = ( - "drv" if self._case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + "med" if self._case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" ) self._ninja = False self._dry_run = False @@ -289,10 +290,12 @@ def run(self, skip_pnl=False): if self._case.get_value("COMPARE_BASELINE"): if do_baseline_ops: self._phase_modifying_call(BASELINE_PHASE, self._compare_baseline) - self._phase_modifying_call(MEMCOMP_PHASE, self._compare_memory) - self._phase_modifying_call( - THROUGHPUT_PHASE, self._compare_throughput - ) + comp_standalone, _ = is_comp_standalone(self._case) + if not comp_standalone: + self._phase_modifying_call(MEMCOMP_PHASE, self._compare_memory) + self._phase_modifying_call( + THROUGHPUT_PHASE, self._compare_throughput + ) else: with self._test_status: self._test_status.set_status(BASELINE_PHASE, TEST_PEND_STATUS) @@ -456,7 +459,7 @@ def run_indv( raise CIMEError( "Could not find all inputdata on any server, try " "manually running `./check_input_data --download " - f"--versbose` from {caseroot!r}." + f"--verbose` from {caseroot!r}." ) from None if submit_resubmits is None: do_resub = self._case.get_value("BATCH_SYSTEM") != "none" @@ -806,7 +809,7 @@ def perf_check_for_memory_leak(case, tolerance): for cpllog in latestcpllogs: try: - memlist = _perf_get_memory(case, cpllog) + memlist = perf_get_memory_list(case, cpllog) except RuntimeError: return False, "insufficient data for memleak test" diff --git a/CIME/SystemTests/system_tests_compare_two.py b/CIME/SystemTests/system_tests_compare_two.py index c58cfa372ba..5eaac4948e1 100644 --- a/CIME/SystemTests/system_tests_compare_two.py +++ b/CIME/SystemTests/system_tests_compare_two.py @@ -24,6 +24,9 @@ (2) _case_two_setup This method will be called to set up case 2, the "test" case +Note that the base class will always call case_setup(reset=True) on +both case1 and case2 during setup. + In addition, they MAY require the following methods: (1) _common_setup @@ -559,6 +562,7 @@ def _setup_cases(self): self._activate_case2() self._common_setup() self._case_two_setup() + self._case2.case_setup(test_mode=True, reset=True) fix_single_exe_case(self._case2) diff --git a/CIME/SystemTests/tsc.py b/CIME/SystemTests/tsc.py index 3ecaefe75d0..1a37ecaac5d 100644 --- a/CIME/SystemTests/tsc.py +++ b/CIME/SystemTests/tsc.py @@ -32,7 +32,9 @@ SIM_LENGTH = 600 # seconds OUT_FREQ = 10 # seconds INSPECT_AT = [300, 450, 600] # seconds -INIT_COND_FILE_TEMPLATE = "20210915.v2.ne4_oQU240.F2010.{}.{}.0002-{:02d}-01-00000.nc" +INIT_COND_FILE_TEMPLATE = ( + "20231105.v3b01.F2010.ne4_oQU240.chrysalis.{}.{}.0002-{:02d}-01-00000.nc" +) VAR_LIST = [ "T", "Q", @@ -100,8 +102,8 @@ def _run_with_specified_dtime(self, dtime=2): self._case.set_value("STOP_OPTION", "nsteps") csmdata_root = self._case.get_value("DIN_LOC_ROOT") - csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4_v2_init") - csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4_oQU240_v2_init") + csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4_v3_init") + csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4_oQU240_v3_init") nstep_output = OUT_FREQ // dtime for iinst in range(1, NINST + 1): diff --git a/CIME/Tools/Makefile b/CIME/Tools/Makefile index c663b1edf26..e3ddb9c2042 100644 --- a/CIME/Tools/Makefile +++ b/CIME/Tools/Makefile @@ -48,7 +48,7 @@ ifeq ($(strip $(SMP)),TRUE) THREADDIR = threads compile_threaded = TRUE else - ifeq ($(strip $(SMP_PRESENT)),TRUE) + ifeq ($(strip $(BUILD_THREADED)),TRUE) THREADDIR = threads compile_threaded = TRUE else @@ -373,7 +373,13 @@ ifeq ($(strip $(MPILIB)), mpi-serial) MPIFC := $(SFC) MPICC := $(SCC) MPICXX := $(SCXX) - CONFIG_ARGS += MCT_PATH=$(SHAREDLIBROOT)/$(SHAREDPATH)/mct/mpi-serial + ifndef MPI_SERIAL_PATH + CONFIG_ARGS += MCT_PATH=$(SHAREDLIBROOT)/$(SHAREDPATH)/mct/mpi-serial + else + CONFIG_ARGS += MCT_PATH=$(MPI_SERIAL_PATH) + INC_MPI := $(MPI_SERIAL_PATH)/include + LIB_MPI := $(MPI_SERIAL_PATH)/lib + endif else CC := $(MPICC) FC := $(MPIFC) @@ -567,9 +573,9 @@ ifdef MPAS_LIBDIR # used to build the MPAS dycore if needed. libmpas: cam_abortutils.o physconst.o $(MAKE) -C $(MPAS_LIBDIR) CC="$(CC)" FC="$(FC)" PIODEF="$(PIODEF)" \ - FFLAGS='$(FREEFLAGS) $(FFLAGS)' GPUFLAGS='$(GPUFLAGS)' \ - CASEROOT='$(CASEROOT)' COMPILER='$(COMPILER)' MACH='$(MACH)' \ - FCINCLUDES='$(INCLDIR) $(INCS) -I$(ABS_INSTALL_SHAREDPATH)/include -I$(ABS_ESMF_PATH)/include' + FFLAGS='$(FREEFLAGS) $(FFLAGS)' GPUFLAGS='$(GPUFLAGS)' \ + CASEROOT='$(CASEROOT)' COMPILER='$(COMPILER)' MACH='$(MACH)' \ + FCINCLUDES='$(INCLDIR) $(INCS) -I$(ABS_INSTALL_SHAREDPATH)/include -I$(ABS_ESMF_PATH)/include' dyn_comp.o: libmpas dyn_grid.o: libmpas @@ -595,10 +601,12 @@ ifdef LAPACK_LIBDIR SLIBS += -L$(LAPACK_LIBDIR) -llapack -lblas endif ifdef LIB_MPI - ifndef MPI_LIB_NAME - SLIBS += -L$(LIB_MPI) -lmpi - else - SLIBS += -L$(LIB_MPI) -l$(MPI_LIB_NAME) + ifndef MPI_SERIAL_PATH + ifndef MPI_LIB_NAME + SLIBS += -L$(LIB_MPI) -lmpi + else + SLIBS += -L$(LIB_MPI) -l$(MPI_LIB_NAME) + endif endif endif @@ -919,12 +927,21 @@ GENF90 ?= $(CIMEROOT)/CIME/non_py/externals/genf90/genf90.pl .SUFFIXES: .F90 .F .f90 .f .c .cpp .o .in ifeq ($(MPILIB),mpi-serial) - MPISERIAL = $(INSTALL_SHAREDPATH)/lib/libmpi-serial.a - MLIBS += -L$(INSTALL_SHAREDPATH)/lib -lmpi-serial - CMAKE_OPTS += -DMPI_C_INCLUDE_PATH=$(INSTALL_SHAREDPATH)/include \ + ifdef MPI_SERIAL_PATH + MPISERIAL = $(MPI_SERIAL_PATH)/lib/libmpi-serial.a + MLIBS += -L$(MPI_SERIAL_PATH)/lib -lmpi-serial + CMAKE_OPTS += -DMPI_C_INCLUDE_PATH=$(MPI_SERIAL_PATH)/include \ + -DMPI_Fortran_INCLUDE_PATH=$(MPI_SERIAL_PATH)/include \ + -DMPI_C_LIBRARIES=$(MPI_SERIAL_PATH)/lib/libmpi-serial.a \ + -DMPI_Fortran_LIBRARIES=$(MPI_SERIAL_PATH)/lib/libmpi-serial.a + else + MPISERIAL = $(INSTALL_SHAREDPATH)/lib/libmpi-serial.a + MLIBS += -L$(INSTALL_SHAREDPATH)/lib -lmpi-serial + CMAKE_OPTS += -DMPI_C_INCLUDE_PATH=$(INSTALL_SHAREDPATH)/include \ -DMPI_Fortran_INCLUDE_PATH=$(INSTALL_SHAREDPATH)/include \ -DMPI_C_LIBRARIES=$(INSTALL_SHAREDPATH)/lib/libmpi-serial.a \ -DMPI_Fortran_LIBRARIES=$(INSTALL_SHAREDPATH)/lib/libmpi-serial.a + endif endif $(MCTLIBS) : $(MPISERIAL) diff --git a/CIME/Tools/jenkins_generic_job b/CIME/Tools/jenkins_generic_job index 9787aedab13..ec93bfca238 100755 --- a/CIME/Tools/jenkins_generic_job +++ b/CIME/Tools/jenkins_generic_job @@ -38,7 +38,7 @@ OR CIME.utils.setup_standard_logging_options(parser) - default_baseline = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root()) + default_baseline = CIME.utils.get_current_branch(repo=CIME.utils.get_src_root()) if default_baseline is not None: default_baseline = default_baseline.replace(".", "_").replace( "/", "_" diff --git a/CIME/XML/env_batch.py b/CIME/XML/env_batch.py index 7682ab6e1ee..19d578a389a 100644 --- a/CIME/XML/env_batch.py +++ b/CIME/XML/env_batch.py @@ -813,6 +813,7 @@ def submit_jobs( dry_run=dry_run, workflow=workflow, ) + batch_job_id = str(alljobs.index(job)) if dry_run else result depid[job] = batch_job_id jobcmds.append((job, result)) @@ -921,35 +922,37 @@ def _submit_single_job( logger.info("Starting job script {}".format(job)) function_name = job.replace(".", "_") job_name = "." + job - if not dry_run: - args = self._build_run_args( - job, - True, - skip_pnl=skip_pnl, - set_continue_run=resubmit_immediate, - submit_resubmits=workflow and not resubmit_immediate, - ) - try: - if hasattr(case, function_name): - getattr(case, function_name)( - **{k: v for k, (v, _) in args.items()} - ) + args = self._build_run_args( + job, + True, + skip_pnl=skip_pnl, + set_continue_run=resubmit_immediate, + submit_resubmits=workflow and not resubmit_immediate, + ) + + try: + if hasattr(case, function_name): + if dry_run: + return + + getattr(case, function_name)(**{k: v for k, (v, _) in args.items()}) + else: + expect( + os.path.isfile(job_name), + "Could not find file {}".format(job_name), + ) + if dry_run: + return os.path.join(self._caseroot, job_name) else: - expect( - os.path.isfile(job_name), - "Could not find file {}".format(job_name), - ) run_cmd_no_fail( os.path.join(self._caseroot, job_name), combine_output=True, verbose=True, from_dir=self._caseroot, ) - except Exception as e: - # We don't want exception from the run phases getting into submit phase - logger.warning( - "Exception from {}: {}".format(function_name, str(e)) - ) + except Exception as e: + # We don't want exception from the run phases getting into submit phase + logger.warning("Exception from {}: {}".format(function_name, str(e))) return @@ -1088,10 +1091,10 @@ def _submit_single_job( # add ` before cd $CASEROOT and at end of command submitcmd = submitcmd.replace("cd $CASEROOT", "'cd $CASEROOT") + "'" + submitcmd = case.get_resolved_value(submitcmd, subgroup=job) if dry_run: return submitcmd else: - submitcmd = case.get_resolved_value(submitcmd) logger.info("Submitting job script {}".format(submitcmd)) output = run_cmd_no_fail(submitcmd, combine_output=True) jobid = self.get_job_id(output) @@ -1125,8 +1128,14 @@ def get_job_id(self, output): jobid_pattern is not None, "Could not find jobid_pattern in env_batch.xml", ) + + # If no output was provided, skip the search. This could + # be because --no-batch was provided. + if not output: + return output else: return output + search_match = re.search(jobid_pattern, output) expect( search_match is not None, diff --git a/CIME/XML/env_mach_specific.py b/CIME/XML/env_mach_specific.py index 4652f2a7d0a..0592921ab9a 100644 --- a/CIME/XML/env_mach_specific.py +++ b/CIME/XML/env_mach_specific.py @@ -277,19 +277,25 @@ def make_env_mach_specific_file(self, shell, case, output_dir=""): if env_value.startswith("sh"): lines.append("{}".format(env_name)) else: - lines.append("export {}={}".format(env_name, env_value)) + if env_value is None: + lines.append("unset {}".format(env_name)) + else: + lines.append("export {}={}".format(env_name, env_value)) elif shell == "csh": if env_name == "source": if env_value.startswith("csh"): lines.append("{}".format(env_name)) else: - lines.append("setenv {} {}".format(env_name, env_value)) + if env_value is None: + lines.append("unsetenv {}".format(env_name)) + else: + lines.append("setenv {} {}".format(env_name, env_value)) else: expect(False, "Unknown shell type: '{}'".format(shell)) with open(os.path.join(output_dir, filename), "w") as fd: - fd.write("\n".join(lines)) + fd.write("\n".join(lines) + "\n") # Private API diff --git a/CIME/XML/env_workflow.py b/CIME/XML/env_workflow.py index 3c976693639..c59ff23aba4 100644 --- a/CIME/XML/env_workflow.py +++ b/CIME/XML/env_workflow.py @@ -112,7 +112,13 @@ def get_job_specs(self, case, job): if ngpus_per_node > max_gpus_per_node: ngpus_per_node = max_gpus_per_node - return task_count, num_nodes, tasks_per_node, thread_count, ngpus_per_node + return ( + task_count, + num_nodes, + tasks_per_node, + thread_count, + ngpus_per_node, + ) # pylint: disable=arguments-differ def get_value(self, item, attribute=None, resolved=True, subgroup="PRIMARY"): diff --git a/CIME/XML/files.py b/CIME/XML/files.py index 26843409ed0..c0149f9601a 100644 --- a/CIME/XML/files.py +++ b/CIME/XML/files.py @@ -136,7 +136,9 @@ def set_value(self, vid, value, subgroup=None, ignore_type=False): def get_schema(self, nodename, attributes=None): node = self.get_optional_child("entry", {"id": nodename}) + schemanode = self.get_optional_child("schema", root=node, attributes=attributes) + if schemanode is not None: logger.debug("Found schema for {}".format(nodename)) return self.get_resolved_value(self.text(schemanode)) diff --git a/CIME/XML/generic_xml.py b/CIME/XML/generic_xml.py index a45ca766ee7..083743695b3 100644 --- a/CIME/XML/generic_xml.py +++ b/CIME/XML/generic_xml.py @@ -8,7 +8,7 @@ import xml.etree.ElementTree as ET # pylint: disable=import-error -from distutils.spawn import find_executable +from shutil import which import getpass from copy import deepcopy from collections import namedtuple @@ -105,7 +105,8 @@ def __init__( def read(self, infile, schema=None): """ - Read and parse an xml file into the object + Read and parse an xml file into the object. The schema variable can either be a path to an xsd schema file or + a dictionary of paths to files by version. """ cached_read = False if not self.DISABLE_CACHING and infile in self._FILEMAP: @@ -126,8 +127,10 @@ def read(self, infile, schema=None): logger.debug("read: {}".format(infile)) with open(infile, "r", encoding="utf-8") as fd: self.read_fd(fd) - - if schema is not None and self.get_version() > 1.0: + version = str(self.get_version()) + if type(schema) is dict: + self.validate_xml_file(infile, schema[version]) + elif schema is not None and self.get_version() > 1.0: self.validate_xml_file(infile, schema) logger.debug("File version is {}".format(str(self.get_version()))) @@ -472,7 +475,7 @@ def write(self, outfile=None, force_write=False): xmlstr = self.get_raw_record() # xmllint provides a better format option for the output file - xmllint = find_executable("xmllint") + xmllint = which("xmllint") if xmllint: if isinstance(outfile, str): @@ -609,7 +612,9 @@ def set_value( return value if valnodes else None - def get_resolved_value(self, raw_value, allow_unresolved_envvars=False): + def get_resolved_value( + self, raw_value, allow_unresolved_envvars=False, subgroup=None + ): """ A value in the xml file may contain references to other xml variables or to environment variables. These are refered to in @@ -659,7 +664,8 @@ def get_resolved_value(self, raw_value, allow_unresolved_envvars=False): logger.debug("find: {}".format(var)) # The overridden versions of this method do not simply return None # so the pylint should not be flagging this - ref = self.get_value(var) # pylint: disable=assignment-from-none + # pylint: disable=assignment-from-none + ref = self.get_value(var, subgroup=subgroup) if ref is not None: logger.debug("resolve: " + str(ref)) @@ -688,9 +694,14 @@ def validate_xml_file(self, filename, schema): """ validate an XML file against a provided schema file using pylint """ - expect(os.path.isfile(filename), "xml file not found {}".format(filename)) - expect(os.path.isfile(schema), "schema file not found {}".format(schema)) - xmllint = find_executable("xmllint") + expect( + filename and os.path.isfile(filename), + "xml file not found {}".format(filename), + ) + expect( + schema and os.path.isfile(schema), "schema file not found {}".format(schema) + ) + xmllint = which("xmllint") expect( xmllint and os.path.isfile(xmllint), diff --git a/CIME/XML/machines.py b/CIME/XML/machines.py index 1b45cf5b580..e3a047d25de 100644 --- a/CIME/XML/machines.py +++ b/CIME/XML/machines.py @@ -7,12 +7,20 @@ from CIME.utils import convert_to_unknown_type, get_cime_config import socket +from pathlib import Path logger = logging.getLogger(__name__) class Machines(GenericXML): - def __init__(self, infile=None, files=None, machine=None, extra_machines_dir=None): + def __init__( + self, + infile=None, + files=None, + machine=None, + extra_machines_dir=None, + read_only=True, + ): """ initialize an object if a filename is provided it will be used, @@ -23,6 +31,9 @@ def __init__(self, infile=None, files=None, machine=None, extra_machines_dir=Non additional directory that will be searched for a config_machines.xml file; if found, the contents of this file will be appended to the standard config_machines.xml. An empty string is treated the same as None. + + The schema variable can be passed as a path to an xsd schema file or a dictionary of paths + with version number as keys. """ self.machine_node = None @@ -37,8 +48,6 @@ def __init__(self, infile=None, files=None, machine=None, extra_machines_dir=Non files = Files() if infile is None: infile = files.get_value("MACHINES_SPEC_FILE") - schema = files.get_schema("MACHINES_SPEC_FILE") - logger.debug("Verifying using schema {}".format(schema)) self.machines_dir = os.path.dirname(infile) if os.path.exists(infile): @@ -46,7 +55,21 @@ def __init__(self, infile=None, files=None, machine=None, extra_machines_dir=Non else: expect(False, f"file not found {infile}") - GenericXML.__init__(self, infile, schema) + schema = { + "3.0": files.get_schema( + "MACHINES_SPEC_FILE", attributes={"version": "3.0"} + ), + "2.0": files.get_schema( + "MACHINES_SPEC_FILE", attributes={"version": "2.0"} + ), + } + # Before v3 there was but one choice + if not schema["3.0"]: + schema = files.get_schema("MACHINES_SPEC_FILE") + + logger.debug("Verifying using schema {}".format(schema)) + + GenericXML.__init__(self, infile, schema, read_only=read_only) # Append the contents of $HOME/.cime/config_machines.xml if it exists. # @@ -84,7 +107,7 @@ def __init__(self, infile=None, files=None, machine=None, extra_machines_dir=Non machine is not None, f"Could not initialize machine object from {', '.join(checked_files)}. This machine is not available for the target CIME_MODEL.", ) - self.set_machine(machine) + self.set_machine(machine, schema=schema) def get_child(self, name=None, attributes=None, root=None, err_msg=None): if root is None: @@ -132,6 +155,19 @@ def list_available_machines(self): for node in nodes: mach = self.get(node, "MACH") machines.append(mach) + if self.get_version() == 3.0: + machdirs = [ + os.path.basename(f.path) + for f in os.scandir(self.machines_dir) + if f.is_dir() + ] + machdirs.remove("cmake_macros") + machdirs.remove("userdefined_laptop_template") + for mach in machdirs: + if mach not in machines: + machines.append(mach) + + machines.sort() return machines def probe_machine_name(self, warn=True): @@ -143,6 +179,7 @@ def probe_machine_name(self, warn=True): names_not_found = [] nametomatch = socket.getfqdn() + machine = self._probe_machine_name_one_guess(nametomatch) if machine is None: @@ -170,10 +207,15 @@ def _probe_machine_name_one_guess(self, nametomatch): Find a matching regular expression for nametomatch in the NODENAME_REGEX field in the file. First match wins. Returns None if no match is found. """ + if self.get_version() < 3: + return self._probe_machine_name_one_guess_v2(nametomatch) + else: + return self._probe_machine_name_one_guess_v3(nametomatch) - machine = None - nodes = self.get_children("machine") + def _probe_machine_name_one_guess_v2(self, nametomatch): + nodes = self.get_children("machine") + machine = None for node in nodes: machtocheck = self.get(node, "MACH") logger.debug("machine is " + machtocheck) @@ -215,7 +257,55 @@ def _probe_machine_name_one_guess(self, nametomatch): return machine - def set_machine(self, machine): + def _probe_machine_name_one_guess_v3(self, nametomatch): + + nodes = self.get_children("NODENAME_REGEX", root=self.root) + + children = [y for x in nodes for y in self.get_children(root=x)] + + for child in children: + machtocheck = self.get(child, "MACH") + regex_str = self.text(child) + logger.debug( + "machine is {} regex {}, nametomatch {}".format( + machtocheck, regex_str, nametomatch + ) + ) + + if regex_str is not None: + # an environment variable can be used + if regex_str.startswith("$ENV"): + machine_value = self.get_resolved_value( + regex_str, allow_unresolved_envvars=True + ) + logger.debug("machine_value is {}".format(machine_value)) + if not machine_value.startswith("$ENV"): + try: + match, this_machine = machine_value.split(":") + except ValueError: + expect( + False, + "Bad formation of NODENAME_REGEX. Expected envvar:value, found {}".format( + regex_str + ), + ) + if match == this_machine: + machine = machtocheck + break + else: + regex = re.compile(regex_str) + if regex.match(nametomatch): + logger.debug( + "Found machine: {} matches {}".format( + machtocheck, nametomatch + ) + ) + machine = machtocheck + break + + return machine + + def set_machine(self, machine, schema=None): """ Sets the machine block in the Machines object @@ -228,15 +318,34 @@ def set_machine(self, machine): CIMEError: ERROR: No machine trump found """ if machine == "Query": - self.machine = machine - elif self.machine != machine or self.machine_node is None: - self.machine_node = super(Machines, self).get_child( - "machine", - {"MACH": machine}, - err_msg="No machine {} found".format(machine), - ) - self.machine = machine + return machine + elif self.get_version() == 3: + machines_file = Path.home() / ".cime" / machine / "config_machines.xml" + + if machines_file.exists(): + GenericXML.read( + self, + machines_file, + schema=schema, + ) + else: + machines_file = ( + Path(self.machines_dir) / machine / "config_machines.xml" + ) + + if machines_file.exists(): + GenericXML.read( + self, + machines_file, + schema=schema, + ) + self.machine_node = super(Machines, self).get_child( + "machine", + {"MACH": machine}, + err_msg="No machine {} found".format(machine), + ) + self.machine = machine return machine # pylint: disable=arguments-differ @@ -285,6 +394,11 @@ def get_field_from_list(self, listname, reqval=None, attributes=None): """ expect(self.machine_node is not None, "Machine object has no machine defined") supported_values = self.get_value(listname, attributes=attributes) + logger.debug( + "supported values for {} on {} is {}".format( + listname, self.machine, supported_values + ) + ) # if no match with attributes, try without if supported_values is None: supported_values = self.get_value(listname, attributes=None) diff --git a/CIME/baselines/performance.py b/CIME/baselines/performance.py index fe940309b28..55092f397e2 100644 --- a/CIME/baselines/performance.py +++ b/CIME/baselines/performance.py @@ -121,25 +121,25 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True): if throughput: try: - tput = perf_get_throughput(case, config) + tput, mode = perf_get_throughput(case, config) except RuntimeError as e: logger.debug("Could not get throughput: {0!s}".format(e)) else: baseline_file = os.path.join(basegen_dir, "cpl-tput.log") - write_baseline_file(baseline_file, tput) + write_baseline_file(baseline_file, tput, mode) logger.info("Updated throughput baseline to {!s}".format(tput)) if memory: try: - mem = perf_get_memory(case, config) + mem, mode = perf_get_memory(case, config) except RuntimeError as e: logger.info("Could not get memory usage: {0!s}".format(e)) else: baseline_file = os.path.join(basegen_dir, "cpl-mem.log") - write_baseline_file(baseline_file, mem) + write_baseline_file(baseline_file, mem, mode) logger.info("Updated memory usage baseline to {!s}".format(mem)) @@ -184,16 +184,11 @@ def perf_get_throughput(case, config): Model throughput. """ try: - tput = config.perf_get_throughput(case) + tput, mode = config.perf_get_throughput(case) except AttributeError: - tput = _perf_get_throughput(case) + tput, mode = _perf_get_throughput(case) - if tput is None: - raise RuntimeError("Could not get default throughput") from None - - tput = str(tput) - - return tput + return tput, mode def perf_get_memory(case, config): @@ -215,19 +210,14 @@ def perf_get_memory(case, config): Model memory usage. """ try: - mem = config.perf_get_memory(case) + mem, mode = config.perf_get_memory(case) except AttributeError: - mem = _perf_get_memory(case) - - if mem is None: - raise RuntimeError("Could not get default memory usage") from None - - mem = str(mem[-1][1]) + mem, mode = _perf_get_memory(case) - return mem + return mem, mode -def write_baseline_file(baseline_file, value): +def write_baseline_file(baseline_file, value, mode="a"): """ Writes value to `baseline_file`. @@ -237,13 +227,10 @@ def write_baseline_file(baseline_file, value): Path to the baseline file. value : str Value to write. + mode : str + Mode to open file with. """ - commit_hash = get_current_commit(repo=get_src_root()) - - timestamp = get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S") - - with open(baseline_file, "w") as fd: - fd.write(f"# sha:{commit_hash} date: {timestamp}\n") + with open(baseline_file, mode) as fd: fd.write(value) @@ -270,6 +257,17 @@ def _perf_get_memory(case, cpllog=None): RuntimeError If not enough sample were found. """ + memlist = perf_get_memory_list(case, cpllog) + + if memlist is None: + raise RuntimeError("Could not get default memory usage") from None + + value = _format_baseline(memlist[-1][1]) + + return value, "a" + + +def perf_get_memory_list(case, cpllog): if cpllog is None: cpllog = get_latest_cpl_logs(case) else: @@ -317,7 +315,12 @@ def _perf_get_throughput(case): logger.debug("Could not parse throughput from coupler log") - return tput + if tput is None: + raise RuntimeError("Could not get default throughput") from None + + value = _format_baseline(tput) + + return value, "a" def get_latest_cpl_logs(case): @@ -326,7 +329,7 @@ def get_latest_cpl_logs(case): """ coupler_log_path = case.get_value("RUNDIR") - cpllog_name = "drv" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" + cpllog_name = "med" if case.get_value("COMP_INTERFACE") == "nuopc" else "cpl" cpllogs = glob.glob(os.path.join(coupler_log_path, "{}*.log.*".format(cpllog_name))) @@ -429,7 +432,7 @@ def read_baseline_file(baseline_file): Value stored in baseline file without comments. """ with open(baseline_file) as fd: - lines = [x.strip() for x in fd.readlines() if not x.startswith("#")] + lines = [x.strip() for x in fd.readlines() if not x.startswith("#") and x != ""] return "\n".join(lines) @@ -456,13 +459,20 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): comment : str provides explanation from comparison. """ - current = _perf_get_throughput(case) + current, _ = _perf_get_throughput(case) + + try: + current = float(_parse_baseline(current)) + except (ValueError, TypeError): + comment = "Could not compare throughput to baseline, as baseline had no value." + + return None, comment try: # default baseline is stored as single float - baseline = float(baseline) - except ValueError: - comment = "Could not compare throughput to baseline, as basline had no value." + baseline = float(_parse_baseline(baseline)) + except (ValueError, TypeError): + comment = "Could not compare throughput to baseline, as baseline had no value." return None, comment @@ -474,14 +484,13 @@ def _perf_compare_throughput_baseline(case, baseline, tolerance): if diff is not None: below_tolerance = diff < tolerance + info = "Throughput changed by {:.2f}%: baseline={:.3f} sypd, tolerance={:d}%, current={:.3f} sypd".format( + diff * 100, baseline, int(tolerance * 100), current + ) if below_tolerance: - comment = "TPUTCOMP: Computation time changed by {:.2f}% relative to baseline".format( - diff * 100 - ) + comment = "TPUTCOMP: " + info else: - comment = "Error: TPUTCOMP: Computation time increase > {:d}% from baseline".format( - int(tolerance * 100) - ) + comment = "Error: TPUTCOMP: " + info return below_tolerance, comment @@ -509,16 +518,21 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): provides explanation from comparison. """ try: - current = _perf_get_memory(case) + current, _ = _perf_get_memory(case) except RuntimeError as e: return None, str(e) - else: - current = current[-1][1] + + try: + current = float(_parse_baseline(current)) + except (ValueError, TypeError): + comment = "Could not compare throughput to baseline, as baseline had no value." + + return None, comment try: # default baseline is stored as single float - baseline = float(baseline) - except ValueError: + baseline = float(_parse_baseline(baseline)) + except (ValueError, TypeError): baseline = 0.0 try: @@ -533,13 +547,64 @@ def _perf_compare_memory_baseline(case, baseline, tolerance): if diff is not None: below_tolerance = diff < tolerance + info = "Memory usage highwater changed by {:.2f}%: baseline={:.3f} MB, tolerance={:d}%, current={:.3f} MB".format( + diff * 100, baseline, int(tolerance * 100), current + ) if below_tolerance: - comment = "MEMCOMP: Memory usage highwater has changed by {:.2f}% relative to baseline".format( - diff * 100 - ) + comment = "MEMCOMP: " + info else: - comment = "Error: Memory usage increase >{:d}% from baseline's {:f} to {:f}".format( - int(tolerance * 100), baseline, current - ) + comment = "Error: MEMCOMP: " + info return below_tolerance, comment + + +def _format_baseline(value): + """ + Encodes value with default baseline format. + + Default format: + sha: date: + + Parameters + ---------- + value : str + Baseline value to encode. + + Returns + ------- + value : str + Baseline entry. + """ + commit_hash = get_current_commit(repo=get_src_root()) + + timestamp = get_timestamp(timestamp_format="%Y-%m-%d_%H:%M:%S") + + return f"sha:{commit_hash} date:{timestamp} {value}\n" + + +def _parse_baseline(data): + """ + Parses default baseline format. + + Default format: + sha: date: + + Parameters + ---------- + data : str + Containing contents of baseline file. + + Returns + ------- + value : str + Value of the latest blessed baseline. + """ + lines = data.split("\n") + lines = [x for x in lines if x != ""] + + try: + value = lines[-1].strip().split(" ")[-1] + except IndexError: + value = None + + return value diff --git a/CIME/bless_test_results.py b/CIME/bless_test_results.py index c6f0754ebd8..0502c541d3a 100644 --- a/CIME/bless_test_results.py +++ b/CIME/bless_test_results.py @@ -216,7 +216,7 @@ def bless_test_results( bless_perf=False, **_, # Capture all for extra ): - bless_all = not (namelists_only | hist_only) + bless_all = not (namelists_only | hist_only | bless_tput | bless_mem | bless_perf) test_status_files = get_test_status_files(test_root, compiler, test_id=test_id) diff --git a/CIME/build.py b/CIME/build.py index b8d481b80d8..3f5c57ca998 100644 --- a/CIME/build.py +++ b/CIME/build.py @@ -45,7 +45,7 @@ "OS", "PIO_VERSION", "SHAREDLIBROOT", - "SMP_PRESENT", + "BUILD_THREADED", "USE_ESMF_LIB", "USE_MOAB", "CAM_CONFIG_OPTS", diff --git a/CIME/build_scripts/buildlib.cprnc b/CIME/build_scripts/buildlib.cprnc index 5e6708da133..51426de27c6 100755 --- a/CIME/build_scripts/buildlib.cprnc +++ b/CIME/build_scripts/buildlib.cprnc @@ -6,7 +6,7 @@ sys.path.append(os.path.join(_CIMEROOT, "CIME", "Tools")) from standard_script_setup import * from CIME import utils -from CIME.utils import run_bld_cmd_ensure_logging +from CIME.utils import run_bld_cmd_ensure_logging, CIMEError from CIME.case import Case from CIME.build import get_standard_cmake_args @@ -63,10 +63,35 @@ def buildlib(bldroot, installpath, case): ) cmake_args = get_standard_cmake_args(case, "ignore_sharedpath") + os.environ["CIMEROOT"] = cimeroot - cmake_cmd = ". ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cimeroot}/CIME/non_py/cprnc -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".format( - cimeroot=cimeroot, dest_path=installpath, cmake_args=cmake_args, bldroot=bldroot + + srcroot = case.get_value("SRCROOT") + + cprnc_src_root = None + candidate_paths = ( + os.path.join(cimeroot, "CIME/non_py/cprnc"), + os.path.join(srcroot, "externals/cprnc"), ) + + for candidate in candidate_paths: + if os.path.exists(candidate): + cprnc_src_root = candidate + + break + else: + logger.debug("{!r} is not a valid cprnc source path") + + if cprnc_src_root is None: + raise CIMEError("Could not find a valid cprnc source directory") + + cmake_cmd = ". ./.env_mach_specific.sh && NETCDF=$(dirname $(dirname $(which nf-config))) cmake {cmake_args} -DMPILIB=mpi-serial -DDEBUG=FALSE -C Macros.cmake {cprnc_src_root} -DCMAKE_PREFIX_PATH={dest_path} -DBLDROOT={bldroot}".format( + cprnc_src_root=cprnc_src_root, + dest_path=installpath, + cmake_args=cmake_args, + bldroot=bldroot, + ) + run_bld_cmd_ensure_logging(cmake_cmd, logger, from_dir=bldroot) gmake_cmd = case.get_value("GMAKE") diff --git a/CIME/build_scripts/buildlib.mpi-serial b/CIME/build_scripts/buildlib.mpi-serial index 83ad88367fd..7aaad973d80 100755 --- a/CIME/build_scripts/buildlib.mpi-serial +++ b/CIME/build_scripts/buildlib.mpi-serial @@ -50,6 +50,11 @@ def buildlib(bldroot, installpath, case): ############################################################################### caseroot = case.get_value("CASEROOT") srcroot = case.get_value("SRCROOT") + # check to see if MPI_SERIAL is installed + with open(os.path.join(caseroot, "Macros.make"), "r") as f: + for line in f: + if "MPI_SERIAL_PATH" in line: + return customize_path = os.path.join(srcroot, "cime_config", "customize") diff --git a/CIME/case/case.py b/CIME/case/case.py index 2bf14540205..e08b5ffe2c4 100644 --- a/CIME/case/case.py +++ b/CIME/case/case.py @@ -481,7 +481,7 @@ def get_value(self, item, attribute=None, resolved=True, subgroup=None): if result is not None: if resolved and isinstance(result, str): - result = self.get_resolved_value(result) + result = self.get_resolved_value(result, subgroup=subgroup) vtype = env_file.get_type_info(item) if vtype is not None and vtype != "char": result = convert_to_type(result, vtype, item) @@ -548,13 +548,17 @@ def get_type_info(self, item): return result - def get_resolved_value(self, item, recurse=0, allow_unresolved_envvars=False): + def get_resolved_value( + self, item, recurse=0, allow_unresolved_envvars=False, subgroup=None + ): num_unresolved = item.count("$") if item else 0 recurse_limit = 10 if num_unresolved > 0 and recurse < recurse_limit: for env_file in self._env_entryid_files: item = env_file.get_resolved_value( - item, allow_unresolved_envvars=allow_unresolved_envvars + item, + allow_unresolved_envvars=allow_unresolved_envvars, + subgroup=subgroup, ) if "$" not in item: return item @@ -563,6 +567,7 @@ def get_resolved_value(self, item, recurse=0, allow_unresolved_envvars=False): item, recurse=recurse + 1, allow_unresolved_envvars=allow_unresolved_envvars, + subgroup=subgroup, ) return item diff --git a/CIME/case/case_clone.py b/CIME/case/case_clone.py index 737d26564b3..7b81e0e91b5 100644 --- a/CIME/case/case_clone.py +++ b/CIME/case/case_clone.py @@ -107,7 +107,7 @@ def create_clone( if exeroot is not None: expect( not keepexe, - "create_case_clone: if keepexe is True, " "then exeroot cannot be set", + "create_case_clone: if keepexe is True, then exeroot cannot be set", ) newcase.set_value("EXEROOT", exeroot) if rundir is not None: @@ -219,8 +219,6 @@ def create_clone( ) ) - newcase.case_setup() - return newcase diff --git a/CIME/case/case_run.py b/CIME/case/case_run.py index b7518090504..9dd92ec4876 100644 --- a/CIME/case/case_run.py +++ b/CIME/case/case_run.py @@ -5,11 +5,13 @@ from CIME.config import Config from CIME.utils import gzip_existing_file, new_lid, run_and_log_case_status from CIME.utils import run_sub_or_cmd, append_status, safe_copy, model_log, CIMEError -from CIME.utils import get_model, batch_jobid +from CIME.utils import batch_jobid, is_comp_standalone from CIME.get_timing import get_timing import shutil, time, sys, os, glob +TERMINATION_TEXT = ("HAS ENDED", "END OF MODEL RUN", "SUCCESSFUL TERMINATION") + logger = logging.getLogger(__name__) ############################################################################### @@ -292,19 +294,15 @@ def _post_run_check(case, lid): ############################################################################### rundir = case.get_value("RUNDIR") - model = case.get_value("MODEL") driver = case.get_value("COMP_INTERFACE") - model = get_model() - fv3_standalone = False + comp_standalone, model = is_comp_standalone(case) - if "CPL" not in case.get_values("COMP_CLASSES"): - fv3_standalone = True if driver == "nuopc": - if fv3_standalone: + if comp_standalone: file_prefix = model else: - file_prefix = "drv" + file_prefix = "med" else: file_prefix = "cpl" @@ -322,7 +320,6 @@ def _post_run_check(case, lid): cpl_logs = [os.path.join(rundir, file_prefix + ".log." + lid)] cpl_logfile = cpl_logs[0] - # find the last model.log and cpl.log model_logfile = os.path.join(rundir, model + ".log." + lid) if not os.path.isfile(model_logfile): @@ -332,15 +329,13 @@ def _post_run_check(case, lid): else: count_ok = 0 for cpl_logfile in cpl_logs: - print(f"cpl_logfile {cpl_logfile}") if not os.path.isfile(cpl_logfile): break with open(cpl_logfile, "r") as fd: - if fv3_standalone and "HAS ENDED" in fd.read(): - count_ok += 1 - elif not fv3_standalone and "SUCCESSFUL TERMINATION" in fd.read(): + logfile = fd.read() + if any([x in logfile for x in TERMINATION_TEXT]): count_ok += 1 - if count_ok != cpl_ninst: + if count_ok < cpl_ninst: expect(False, "Model did not complete - see {} \n ".format(cpl_logfile)) diff --git a/CIME/case/case_setup.py b/CIME/case/case_setup.py index 841e90391d0..a170d1bfddd 100644 --- a/CIME/case/case_setup.py +++ b/CIME/case/case_setup.py @@ -28,6 +28,7 @@ logger = logging.getLogger(__name__) + ############################################################################### def _build_usernl_files(case, model, comp): ############################################################################### @@ -344,7 +345,7 @@ def _case_setup_impl( case.initialize_derived_attributes() - case.set_value("SMP_PRESENT", case.get_build_threaded()) + case.set_value("BUILD_THREADED", case.get_build_threaded()) else: case.check_pelayouts_require_rebuild(models) @@ -360,7 +361,7 @@ def _case_setup_impl( cost_per_node = case.get_value("COSTPES_PER_NODE") case.set_value("COST_PES", case.num_nodes * cost_per_node) threaded = case.get_build_threaded() - case.set_value("SMP_PRESENT", threaded) + case.set_value("BUILD_THREADED", threaded) if threaded and case.total_tasks * case.thread_count > cost_per_node: smt_factor = max( 1.0, int(case.get_value("MAX_TASKS_PER_NODE") / cost_per_node) @@ -421,6 +422,15 @@ def _case_setup_impl( run_cmd_no_fail( "{}/cime_config/cism.template {}".format(glcroot, caseroot) ) + if comp == "cam": + camroot = case.get_value("COMP_ROOT_DIR_ATM") + if os.path.exists(os.path.join(camroot, "cam.case_setup.py")): + logger.debug("Running cam.case_setup.py") + run_cmd_no_fail( + "python {cam}/cime_config/cam.case_setup.py {cam} {case}".format( + cam=camroot, case=caseroot + ) + ) _build_usernl_files(case, "drv", "cpl") diff --git a/CIME/case/case_submit.py b/CIME/case/case_submit.py index 4ddb26e3056..e3f30654814 100644 --- a/CIME/case/case_submit.py +++ b/CIME/case/case_submit.py @@ -12,8 +12,6 @@ from CIME.locked_files import unlock_file, lock_file from CIME.test_status import * -import socket - logger = logging.getLogger(__name__) @@ -39,6 +37,7 @@ def _submit( batch_args=None, workflow=True, chksum=False, + dryrun=False, ): if job is None: job = case.get_first_job() @@ -164,9 +163,6 @@ def _submit( case.check_case(skip_pnl=skip_pnl, chksum=chksum) if job == case.get_primary_job(): case.check_DA_settings() - if case.get_value("MACH") == "mira": - with open(".original_host", "w") as fd: - fd.write(socket.gethostname()) # Load Modules case.load_env() @@ -185,16 +181,20 @@ def _submit( mail_type=mail_type, batch_args=batch_args, workflow=workflow, + dry_run=dryrun, ) - xml_jobids = [] - for jobname, jobid in job_ids.items(): - logger.info("Submitted job {} with id {}".format(jobname, jobid)) - if jobid: - xml_jobids.append("{}:{}".format(jobname, jobid)) + if dryrun: + for job in job_ids: + xml_jobids.append("{}:{}".format(job[0], job[1])) + else: + for jobname, jobid in job_ids.items(): + logger.info("Submitted job {} with id {}".format(jobname, jobid)) + if jobid: + xml_jobids.append("{}:{}".format(jobname, jobid)) xml_jobid_text = ", ".join(xml_jobids) - if xml_jobid_text: + if xml_jobid_text and not dryrun: case.set_value("JOB_IDS", xml_jobid_text) return xml_jobid_text @@ -214,6 +214,7 @@ def submit( batch_args=None, workflow=True, chksum=False, + dryrun=False, ): if resubmit_immediate and self.get_value("MACH") in ["mira", "cetus"]: logger.warning( @@ -266,6 +267,7 @@ def submit( batch_args=batch_args, workflow=workflow, chksum=chksum, + dryrun=dryrun, ) run_and_log_case_status( functor, @@ -288,9 +290,9 @@ def check_case(self, skip_pnl=False, chksum=False): self.check_lockedfiles() if not skip_pnl: self.create_namelists() # Must be called before check_all_input_data + logger.info("Checking that inputdata is available as part of case submission") - if not self.get_value("TEST"): - self.check_all_input_data(chksum=chksum) + self.check_all_input_data(chksum=chksum) if self.get_value("COMP_WAV") == "ww": # the ww3 buildnml has dependencies on inputdata so we must run it again @@ -353,7 +355,7 @@ def check_case(self, skip_pnl=False, chksum=False): expect( self.get_value("BUILD_COMPLETE"), - "Build complete is " "not True please rebuild the model by calling case.build", + "Build complete is not True please rebuild the model by calling case.build", ) logger.info("Check case OK") diff --git a/CIME/case/check_input_data.py b/CIME/case/check_input_data.py index d099a6da046..5bff3823d7c 100644 --- a/CIME/case/check_input_data.py +++ b/CIME/case/check_input_data.py @@ -212,7 +212,7 @@ def _check_all_input_data_impl( chksum=chksum and chksum_found, ) if download and not success: - if not chksum: + if chksum: chksum_found = _download_checksum_file(self.get_value("RUNDIR")) success = _downloadfromserver(self, input_data_root, data_list_dir) diff --git a/CIME/data/config/cesm/config_files.xml b/CIME/data/config/cesm/config_files.xml index 3268ccb33f0..adcd79c85e3 100644 --- a/CIME/data/config/cesm/config_files.xml +++ b/CIME/data/config/cesm/config_files.xml @@ -42,7 +42,8 @@ case_last env_case.xml file containing machine specifications for target model primary component (for documentation only - DO NOT EDIT) - $CIMEROOT/CIME/data/config/xml_schemas/config_machines.xsd + $CIMEROOT/CIME/data/config/xml_schemas/config_machines.xsd + $CIMEROOT/CIME/data/config/xml_schemas/config_machines_version3.xsd diff --git a/CIME/data/config/xml_schemas/config_machines_template.xml b/CIME/data/config/xml_schemas/config_machines_template.xml index 99c2bc72581..de361c16265 100644 --- a/CIME/data/config/xml_schemas/config_machines_template.xml +++ b/CIME/data/config/xml_schemas/config_machines_template.xml @@ -1,17 +1,12 @@ - + SITE VENDOR platform, os is ---, xx pes/node, batch system is --- - - .*.cheyenne.ucar.edu - LINUX diff --git a/CIME/data/config/xml_schemas/config_machines_version3.xsd b/CIME/data/config/xml_schemas/config_machines_version3.xsd new file mode 100644 index 00000000000..92b55839fb2 --- /dev/null +++ b/CIME/data/config/xml_schemas/config_machines_version3.xsd @@ -0,0 +1,330 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/CIME/data/config/xml_schemas/env_mach_specific.xsd b/CIME/data/config/xml_schemas/env_mach_specific.xsd index 7778635592b..3c7a3a0d679 100644 --- a/CIME/data/config/xml_schemas/env_mach_specific.xsd +++ b/CIME/data/config/xml_schemas/env_mach_specific.xsd @@ -11,7 +11,7 @@ - + @@ -138,7 +138,7 @@ - + diff --git a/CIME/non_py/cprnc b/CIME/non_py/cprnc new file mode 160000 index 00000000000..9276b219750 --- /dev/null +++ b/CIME/non_py/cprnc @@ -0,0 +1 @@ +Subproject commit 9276b219750881633d8673c72ec80ac821f96d82 diff --git a/CIME/non_py/cprnc/CMakeLists.txt b/CIME/non_py/cprnc/CMakeLists.txt deleted file mode 100644 index 3ec73581ae2..00000000000 --- a/CIME/non_py/cprnc/CMakeLists.txt +++ /dev/null @@ -1,88 +0,0 @@ -# Generate this with: $cimeroot/CIME/scripts/configure --mpilib=mpi-serial --macros-format=CMake -# You'll also need to source the .env_mach_specific.sh file before trying to build cprnc - -include("${BLDROOT}/Macros.cmake") -set(CMAKE_C_COMPILER "${SCC}") -set(CMAKE_Fortran_COMPILER "${SFC}") - -project(CPRNC C Fortran) -enable_language(Fortran) -set(CMAKE_Fortran_FLAGS "${FFLAGS}") - -message("HERE fortran flags are ${CMAKE_Fortran_FLAGS} FFLAGS are ${FFLAGS}") - -cmake_minimum_required(VERSION 2.8) - -# Find netcdf -set(NetCDF_PATH ${NETCDF_PATH}) - -if (EXISTS ${SRC_ROOT}/libraries/parallelio/cmake) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${SRC_ROOT}/libraries/parallelio/cmake) -else() - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${SRC_ROOT}/externals/scorpio/cmake) -endif() - -find_package (NetCDF COMPONENTS Fortran REQUIRED) - -# generate compare_vars_mod.F90 -add_custom_command( - OUTPUT ${PROJECT_BINARY_DIR}/compare_vars_mod.F90 - COMMAND perl ${PROJECT_SOURCE_DIR}/../externals/genf90/genf90.pl - ${PROJECT_SOURCE_DIR}/compare_vars_mod.F90.in > ${PROJECT_BINARY_DIR}/compare_vars_mod.F90 - DEPENDS ${PROJECT_SOURCE_DIR}/compare_vars_mod.F90.in ${PROJECT_SOURCE_DIR}/../externals/genf90/genf90.pl -) - -# Set up includes -include_directories( - ${NetCDF_Fortran_INCLUDE_DIRS} - ${PROJECT_SOURCE_DIR} - ${PROJECT_BINARY_DIR} -) - -# -# Set up lib dependencies, relying on nf-config if possible or NetCDF_Fortran_LIBRARIES -# as a fallback. We want the executable to find libraries via RPATH so that cprnc is -# less-dependent on the current environment (since cprnc is built with a serial netcdf, -# it's likely that a parallel case will have different netcdf modules loaded when it -# comes time to execute cprnc). -# -execute_process(COMMAND ${NETCDF_PATH}/bin/nf-config --flibs - RESULT_VARIABLE NF_CONFIG_RESULT - OUTPUT_VARIABLE NF_CONFIG_OUTPUT) - -if (NF_CONFIG_RESULT STREQUAL "0") - separate_arguments(NF_LIB_LIST UNIX_COMMAND "${NF_CONFIG_OUTPUT}") -else() - set(NF_LIB_LIST ${NetCDF_Fortran_LIBRARIES}) -endif() - -message("lib list is: ${NF_LIB_LIST}") - -foreach(NF_LIB IN LISTS NF_LIB_LIST) - if (NF_LIB MATCHES "-l") - continue() - elseif (NF_LIB MATCHES "-L/") - string(REGEX REPLACE "^-L" "" NF_LIB_DIR "${NF_LIB}") - list(APPEND NF_LIB_DIRS ${NF_LIB_DIR}) - else() - get_filename_component(NF_LIB_DIR ${NF_LIB} DIRECTORY) - list(APPEND NF_LIB_DIRS ${NF_LIB_DIR}) - endif() -endforeach() - -message("lib dirs are: ${NF_LIB_DIRS}") - -set(CMAKE_BUILD_RPATH ${NF_LIB_DIRS}) - -# Add targets -set (CPRNC_SRCS - ${PROJECT_BINARY_DIR}/compare_vars_mod.F90 - filestruct.F90 - utils.F90 - prec.F90 - cprnc.F90 -) - -add_executable(cprnc ${CPRNC_SRCS}) - -target_link_libraries(cprnc ${NF_LIB_LIST}) diff --git a/CIME/non_py/cprnc/Depends b/CIME/non_py/cprnc/Depends deleted file mode 100644 index 582f9ee1794..00000000000 --- a/CIME/non_py/cprnc/Depends +++ /dev/null @@ -1,6 +0,0 @@ -cprnc.o: cprnc.F90 filestruct.o compare_vars_mod.o utils.o -filestruct.o: filestruct.F90 prec.o -prec.o : prec.F90 -compare_vars_mod.o: compare_vars_mod.F90 prec.o utils.o filestruct.o -compare_vars_mod.F90 : compare_vars_mod.F90.in -utils.o : utils.F90 filestruct.o prec.o diff --git a/CIME/non_py/cprnc/Makefile b/CIME/non_py/cprnc/Makefile deleted file mode 100644 index d751cfca03f..00000000000 --- a/CIME/non_py/cprnc/Makefile +++ /dev/null @@ -1,96 +0,0 @@ -#----------------------------------------------------------------------- -# This Makefile is for building cprnc on AIX, Compaq, Linux (with pgf90, -# lf95, ifort, or gfortran compilers), IRIX or SUN platforms. -# -# These macros can be changed by setting environment variables: -# -# Set the path to netcdf: -# -# gmake NETCDF=pathToNetcdf -# -# This sets LIB_NETCDF=$NETCDF/lib and INC_NETCDF=$NETCDF/include -# -# LIB_NETCDF --- Library directory location of netcdf. -# INC_NETCDF --- Include directory location of netcdf. This directory needs to contain -# the NetCDF .mod files, i.e., netcdf.mod and typesizes.mod. -# -# If the include and library files don't have a common root directory then set them -# independently in the commandline: -# -# gmake LIB_NETCDF=pathToLIBFiles INC_NETCDF=pathToINCFiles -# -# You also can set the environment variables: -# -# USER_FC ------ User defined Fortran compiler (for Linux can be pgf90, lf95, ifort, gfortran) -# EXEDIR ------- Directory to build executable in. (Defaults to .) -# VPATH -------- GNU make path. (Defaults to current directory) -# -#------------------------------------------------------------------------ -include Macros.make -# Set up special characters -null := - -EXENAME = cprnc -RM = rm - -NETCDF_PATH ?= $(NETCDF_FORTRAN_PATH) - -# Default for the netcdf library and include directories -LIB_NETCDF := $(NETCDF_PATH)/lib -INC_NETCDF := $(NETCDF_PATH)/include -LDFLAGS = -L$(LIB_NETCDF) -lnetcdff -Wl,-rpath=$(LIB_NETCDF) - -# Determine platform -UNAMES := $(shell uname -s) -SNAME := $(shell uname -n | cut -c1-2) - -GENF90 = ../externals/genf90/genf90.pl - -FC := $(SFC) -FFLAGS += -I$(INC_NETCDF) -I. -#------------------------------------------------------------------------ -# Default rules and macros -#------------------------------------------------------------------------ - -# If path to source code not given -ifeq ($(VPATH),$(null)) - VPATH:= . -endif - -OBJS := compare_vars_mod.o cprnc.o filestruct.o prec.o utils.o - -# If executable directory not given -ifeq ($(EXEDIR),$(null)) - EXEDIR := . -endif - -.SUFFIXES: -.SUFFIXES: .F90 .f90 .o .in - -.F90.o: - $(FC) -c $(FFLAGS) $< - -.f90.o: - $(FC) -c $(FFLAGS) $< - -$(EXEDIR)/$(EXENAME): $(OBJS) - $(FC) -o $@ $(OBJS) $(LDFLAGS) $(SLIBS) - -compare_vars_mod.F90 : compare_vars_mod.F90.in - perl $(GENF90) $< > $@ - -clean: - $(RM) -f $(OBJS) *.mod $(EXEDIR)/$(EXENAME) - -# remove generated file during clean -realclean: - $(RM) -f $(OBJS) *.mod $(EXEDIR)/$(EXENAME) compare_vars_mod.F90 core - -include $(CURDIR)/Depends - -# 'make check' will run the standard tests but without baseline -# comparisons. For complete testing, you should generally also do -# baseline comparisons. See the notes in test_inputs/README for details. -check: $(EXEDIR)/$(EXENAME) - $(RM) -fr tmpdir - ./run_tests -outdir tmpdir diff --git a/CIME/non_py/cprnc/README b/CIME/non_py/cprnc/README deleted file mode 100644 index 6f70818d511..00000000000 --- a/CIME/non_py/cprnc/README +++ /dev/null @@ -1,191 +0,0 @@ -cprnc README ------------- - -cprnc is a generic tool for analyzing a netcdf file or comparing -two netcdf files. - -If you are trying to debug an installed cprnc tool make sure that you -are looking at the correct one by comparing the path to the one in -your case directory. - - -Quick Start Guide: ------------------- - -On cime supported systems you can generate a cmake Macros file using the following -(assuming you are running the command from the directory CIME/non_py/cprnc): - -export CIMEROOT=../.. - -../../scripts/configure --macros-format=CMake --mpilib=mpi-serial --machine=machinename -MPILIB=mpi-serial source ./.env_mach_specific.sh -# Flags if you are using the intel compiler: -CC=icc FC=ifort cmake -DBLDROOT=. -DCASEROOT=. -DCOMPILER=intel -DMACH=cheyenne -DSRC_ROOT=../../../ . -make - -Finally, put the resulting executable in CCSM_CPRNC as defined in -config_machines.xml. - - - Usage: cprnc [-v] [-d dimname:start[:count]] file1 [file2] - -m: Compare each time sample. Default is false, i.e. match "time" - coordinate values before comparing - -v: Verbose output - -d dimname:start[:count] - Print variable values for the specified dimname index subrange. - - -Users Guide: ------------- - -cprnc is a Fortran-90 application. It relies on netcdf version 3 or -later and uses the f90 netcdf interfaces. It requires a netcdf include -file and a netcdf library. - -cprnc generates an ascii output file via standard out. It initially -summarizes some characteristics of the input file[s]. A compare file is -generally 132 characters wide and an analyze file is less than 80 -characters wide. - -In analyze mode, the output for a field looks like - - ( lon, lat, time, -----) - 259200 ( 587, 134, 1) ( 269, 59, 1) - FX1 96369 8.273160400390625E+02 0.000000000000000E+00 - avg abs field values: 9.052845920820910E+01 - -and a guide to this information is printed at the top of the file - - ( dim1, dim2, dim3, dim4) - ARRSIZ1 ( indx1, indx2, indx3) file 1 - FIELD NVALID MAX MIN - - -The first 10 characters of the field name are identified in the first - dozen columns of the third line. -The first line summarizes the names of the dimensions of the field -The second line summarizes the indices of the maximum and minimum value - of the field for the first three dimensions. If the fourth dimension - exists, it's always assumed to be time. Time is handled separately. -The third line summarizes the number of valid values in the array - and the maximum and minimum value over those valid values. Invalid - values are values that are identified to be "fill" value. -The last line summarizes some overall statistics including the average - absolute value of the valid values of the field. - -In comparison mode, the output (132 chars wide) for a field looks like - - 96369 ( lon, lat, time) - 259200 ( 422, 198, 1) ( 203, 186, 1) ( 47, 169, 1) ( 224, 171, 1) - FIRA 96369 1.466549530029297E+02 -3.922052764892578E+01 1.4E+02 -3.037954139709473E+01 1.0E+00 -3.979958057403564E+00 - 96369 1.321966247558594E+02 -1.603044700622559E+01 1.084177169799805E+02 3.982142448425293E+00 - 259200 ( 156, 31, 1) ( 573, 178, 1) ( - avg abs field values: 6.778244097051392E+01 rms diff: 1.4E+01 avg rel diff(npos): 4.6E-02 - 5.960437961084186E+01 avg decimal digits(ndif): 1.2 worst: 0.0 - -and a guide to this information is printed at the top of the file - - NDIFFS ( dim1, dim2, dim3, dim4, ... ) - ARRSIZ1 ( indx1, indx2, indx3, ... ) file 1 - FIELD NVALID1 MAX1 MIN1 DIFFMAX VALUES RDIFMAX VALUES - NVALID2 MAX2 MIN2 - ARRSIZ2 ( indx1, indx2, indx3, ...) file 2 - -The information content is identical to the information in analyze -mode with the following additions. Two additional lines are added -in the main body. Lines 4 and 5 are identical to line 3 and 2 -respectively but are associated with file 2 instead of file 1. -In addition, the right hand side of lines 2, 3, and 4 contain -information about the maximum difference, the location and values -of the maximum difference, the relative difference and the location -and values of the maximum relative difference. The last two line -summarize some overall statistics including average absolute values -of the field on the two files, rms difference, average relative -difference, average number of digits that match, and the worst -case for the number of digits that match. - -"avg rel diff" gives the average relative difference (sum of relative -differences normalized by the number of indices where both variables -have valid values). The denominator for each relative difference is -the MAX of the two values. - -"avg decimal digits" is determined by: For each diff, determine the -number of digits that match (as -log10(rdiff(i)); add this to a -running sum; then normalize by the number of diffs (ignoring places -where the two variables are the same). For example, if there are 10 -values, 8 of which match, one has a relative difference of 1e-3 and -one has a relative difference of 1e-5, then the avg decimal digits -will be 4. - -"worst decimal digits" is simply log10(1/rdmax), where rdmax is the -max relative difference (in the above example, this would give 3). - -At the end of the output file, a summary is presented that looks like - -SUMMARY of cprnc: - A total number of 119 fields were compared - of which 83 had non-zero differences - and 17 had differences in fill patterns - and 2 had differences in dimension sizes - A total number of 10 fields could not be analyzed - A total number of 0 time-varying fields on file 1 were not found on file 2. - A total number of 0 time-constant fields on file 1 were not found on file 2. - A total number of 0 time-varying fields on file 2 were not found on file 1. - A total number of 0 time-constant fields on file 2 were not found on file 1. - diff_test: the two files seem to be DIFFERENT - - -This summarizes: -- the number of fields that were compared -- the number of fields that differed (not counting fields that differed - only in the fill pattern) -- the number of fields with differences in fill patterns -- the number of fields with differences in dimension sizes -- the number of fields that could not be analyzed -- the number of fields on one file but not the other - - for files with an unlimited (time) dimension, these counts are - broken down into time-varying fields (i.e., fields with an unlimited - dimension) and time-constant fields (i.e., fields without an - unlimited dimension) -- whether the files are IDENTICAL, DIFFERENT, or DIFFER only in their field lists - - Files are considered DIFFERENT if there are differences in the values, fill - patterns or dimension sizes of any variable - - Files are considered to "DIFFER only in their field lists" if matching - variables are all identical, but there are either fields on file1 that are - not on file2, or fields on file2 that are not on file1 - - However, if the only difference in field lists is in the presence - or absence of time-constant fields on a file that has an unlimited - (time) dimension, the files are considered to be IDENTICAL, with - an extra message appended that notes this fact. (While not ideal, - this exception is needed so that exact restart tests pass despite - some time-constant fields being on the output files from one case - but not the other.) - -Developers Guide: ------------------ - -The tool works as follows. - -Fields can be analyzed if they are int, float or double and -have between 0 and n dimensions - -In general, fields that appear on both files are -compared. If they are sizes, no difference -statistics are computed and only a summary of the fields on -the files are presented. If fields only appear -on one file, those fields are analyzed. - -The unlimited dimension is treated uniquely. In general, for files -that have a dimension named "time", the time axes are compared -and matching time values on the two files are compared one -timestep at a time. Time values that don't match are skipped. -To override the matching behaviour, use cprnc -m. In this mode, -timestamps are compared in indexical space. In analyze mode, -the fields are analyzed one timestamp at a time. In general, -if there is a "time" axis, it will be the outer-most loop in -the output analysis. In compare mode, fields with a time axis -and a timestamp that are not common between the two files are -ignored. - -It is also possible to compare files that don't have an unlimited -dimension; in this case, the '-m' flag must be given. diff --git a/CIME/non_py/cprnc/compare_vars_mod.F90.in b/CIME/non_py/cprnc/compare_vars_mod.F90.in deleted file mode 100644 index 5b2e1a1966e..00000000000 --- a/CIME/non_py/cprnc/compare_vars_mod.F90.in +++ /dev/null @@ -1,638 +0,0 @@ -module compare_vars_mod - use filestruct, only : file_t, var_t, is_time_varying, vdimsize, dim_t, verbose - use prec, only : r4, r8, i4 - use netcdf, only : nf90_char, nf90_int, nf90_double, nf90_float, nf90_get_var, nf90_max_dims, & - nf90_inq_varid, nf90_get_att, nf90_noerr - use utils, only : checknf90, get_dim_str, get_dimname_str - implicit none - logical :: ignoretime - - interface compute_var_stats -! TYPE real,double,int - module procedure compute_var_stats_{TYPE} - end interface - - interface get_rdiff_stats - ! TYPE real,double - module procedure get_rdiff_stats_{TYPE} - end interface - -contains - - subroutine compare_vars(n,file, vtotal, ndiffs, nfilldiffs, vsizes_differ, & - vnot_analyzed, vtypes_differ ) - integer, intent(in) :: n ! number of files to analyze (1 or 2) - integer, intent(out) :: vtotal - integer, intent(out) :: ndiffs ! number of fields with differences (not counting fields that differ only in the fill pattern) - integer, intent(out) :: nfilldiffs ! number of fields with differences in fill pattern - integer, intent(out) :: vsizes_differ - integer, intent(out) :: vnot_analyzed - integer, intent(out) :: vtypes_differ - - - type(file_t) :: file(n) - double precision, pointer :: time(:,:) - double precision :: tdiff - ! start by making sure that times match - integer :: ierr, ns1, ns2, vid1 - type(var_t), pointer :: v1 - integer :: i, t, nvars, t1, t2, nt - integer :: vidnsteph - integer, allocatable :: nsteph(:) - character(len=132) :: dimstr - type(dim_t), pointer :: udim - real(r8), parameter :: timeepsilon = 1.e-9 ! time diff less than this considered insignificant - - - vtotal = 0 - vsizes_differ = 0 - vtypes_differ = 0 - vnot_analyzed = 0 - - if(n==2 .and. .not.ignoretime) then - ! NOTE(wjs, 2019-03-21) Most of the cprnc code allows the unlimited dimension to be - ! named anything - not necessarily 'time'. But this block of code assumes that the - ! unlimited dimension is named 'time' in order to find the associated coordinate - ! variable. We should probably generalize this by looking for a variable with the - ! same name as the unlimited dimension. - call checknf90(nf90_inq_varid(file(1)%fh, 'time', vid1), & - err_str='These files don''t have a time dimension, use cprnc with -m') - - ns1 = file(1)%dim(file(1)%unlimdimid)%dimsize - if(n==2) then - ns2 = file(2)%dim(file(2)%unlimdimid)%dimsize - else - ns2=1 - end if - allocate(time(max(ns1,ns2),2)) - - call checknf90(nf90_get_var(file(1)%fh, vid1, time(1:ns1,1))) - if(n==2) then - call checknf90(nf90_get_var(file(2)%fh, file(1)%var(vid1)%matchid, time(1:ns2,2))) - end if - if(verbose) then - print *,'File 1 time: ', time(1:ns1,1) - print *,'File 2 time: ', time(1:ns2,2) - end if - end if - - nvars = size(file(1)%var) - if (file(1)%has_unlimited_dim()) then - udim => file(1)%dim(file(1)%unlimdimid) - else - if (.not. ignoretime) then - write(6,*) 'ERROR: For files without an unlimited dimension,' - write(6,*) 'ignore_time needs to be true (via setting the -m flag to cprnc)' - stop - end if - end if - - ndiffs = 0 - nfilldiffs = 0 - -! First look at variables which do not have unlimdim - do i=1,nvars - v1 => file(1)%var(i) - - if (.not. is_time_varying(v1, file(1)%has_unlimited_dim(), file(1)%unlimdimid)) then - call get_dimname_str(v1%ndims,v1%dimids,file(1)%dim,dimstr) - write(6,140) trim(v1%name),trim(dimstr) - vtotal = vtotal+1 - - call compare_one_var(v1=v1, numcases=n, file=file, varnum=i, & - vsizes_differ=vsizes_differ, & - vnot_analyzed=vnot_analyzed, & - vtypes_differ=vtypes_differ, & - ndiffs=ndiffs, nfilldiffs=nfilldiffs) - - end if - end do - -! Now look at variables that DO have unlimdim - if (file(1)%has_unlimited_dim()) then - - ierr = nf90_inq_varid(file(1)%fh, 'nsteph', vidnsteph) - if(ierr == NF90_NOERR) then - allocate(nsteph(udim%kount)) - call checknf90(nf90_get_var(file(1)%fh, vidnsteph, nsteph)) - end if - - - do t=1,udim%dimsize,udim%kount - t1 = t ! need to find mathing times - assumed for now - t2 = t - if(.not. ignoretime) then - do while(t1<=ns1 .and. t2<= ns2) - tdiff = abs(time(t1,1) - time(t2,2)) - if (tdiff < timeepsilon) exit - if(time(t1,1) < time(t2,2)) then - Write(6,*) 'Skipping a time sample on file 1' - t1=t1+1 - else if(time(t1,1) > time(t2,2)) then - Write(6,*) 'Skipping a time sample on file 2' - t2=t2+1 - end if - end do - if(verbose) print *,__FILE__,__LINE__,tdiff,timeepsilon, t1, t2 - if(tdiff< timeepsilon .and. t1/=t2) then - Write(6,*) 'Found common timesteps:', t1, t2 - else if(tdiff > timeepsilon) then - Write(6,*) 'No matching time found.' - vnot_analyzed = nvars - return - end if - if(verbose) print *,__FILE__,__LINE__,time(t1,1),time(t2,2), t1, t2, time(:,:) - end if - - - if(allocated(nsteph)) then - print *,'NSTEPH: ',nsteph(t) - deallocate(nsteph) - end if - - do i=1,nvars - v1 => file(1)%var(i) - if (is_time_varying(v1, file(1)%has_unlimited_dim(), file(1)%unlimdimid)) then - call get_dimname_str(v1%ndims,v1%dimids,file(1)%dim,dimstr) - vtotal = vtotal+1 - write(6,145) trim(v1%name),trim(dimstr), t1, t2 - - call compare_one_var(v1=v1, numcases=n, file=file, varnum=i, & - vsizes_differ=vsizes_differ, & - vnot_analyzed=vnot_analyzed, & - vtypes_differ=vtypes_differ, & - ndiffs=ndiffs, nfilldiffs=nfilldiffs, & - tindex=(/t1, t2/)) - - end if - end do - end do - end if ! if (file(1)%has_unlimited_dim()) - -140 format(1x,a,3x,a) -145 format(1x,a,3x,a,' t_index = ',2i6) - - end subroutine compare_vars - - - ! Compare a single variable, and update counts - ! For variables with multiple time slices, this just does comparisons for a single time slice - subroutine compare_one_var(v1, numcases, file, varnum, & - vsizes_differ, vnot_analyzed, vtypes_differ, & - ndiffs, nfilldiffs, & - tindex) - type(var_t) , intent(in) :: v1 ! variable info for the variable in file 1 - integer , intent(in) :: numcases - type(file_t), intent(in) :: file(numcases) - integer , intent(in) :: varnum - integer , intent(inout) :: vsizes_differ - integer , intent(inout) :: vnot_analyzed - integer , intent(inout) :: vtypes_differ - integer , intent(inout) :: ndiffs - integer , intent(inout) :: nfilldiffs - integer , intent(in), optional :: tindex(numcases) - - integer :: idiff, ifilldiff, isizes_differ, inot_analyzed, itypes_differ - - ! initialize output arguments of compare_var in case compare_var doesn't get called - idiff = 0 - ifilldiff = 0 - isizes_differ = 0 - itypes_differ = 0 - inot_analyzed = 0 - - select case(v1%xtype) - case(nf90_int) - call compare_var_int(numcases,file,(/varnum,v1%matchid/), & - idiff, ifilldiff, isizes_differ, itypes_differ, & - tindex) - case(nf90_float) - call compare_var_real(numcases,file,(/varnum,v1%matchid/), & - idiff, ifilldiff, isizes_differ, itypes_differ, & - tindex) - case(nf90_double) - call compare_var_double(numcases,file,(/varnum,v1%matchid/), & - idiff, ifilldiff, isizes_differ, itypes_differ, & - tindex) - case(nf90_char) - inot_analyzed = 1 - ! call compare_var_char(file1,file2,i,v1%matchid) - case default - print *,'Type not recognized for variable: ', v1%name - end select - - vsizes_differ = vsizes_differ+isizes_differ - vtypes_differ = vtypes_differ+itypes_differ - vnot_analyzed = vnot_analyzed+inot_analyzed - ndiffs = ndiffs+idiff - nfilldiffs = nfilldiffs + ifilldiff - end subroutine compare_one_var - - - ! TYPE real,int,double - subroutine compare_var_{TYPE}(n,file, vid, idiff, ifilldiff, ifail, itypes, tindex) - use, intrinsic :: ieee_arithmetic, only: ieee_is_nan - integer, intent(in) :: n - type(file_t) :: file(2) - integer, intent(in) :: vid(2) - integer, intent(out) :: idiff ! 1 if diffs in field, 0 otherwise (0 if only diffs are in fill pattern) - integer, intent(out) :: ifilldiff ! 1 if diffs in fill pattern, 0 otherwise - ! (idiff & ifilldiff are both 1 if there are diffs in both the fill pattern and the valid values) - integer, intent(out) :: ifail ! 1 if variable sizes differ, 0 otherwise - integer, intent(out) :: itypes ! 1 if variable types differ, 0 otherwise - integer, optional :: tindex(2) - - integer :: s1, s2, l1(1), i, ierr - - {VTYPE}, pointer :: buf(:,:), vdiff(:) - {VTYPE} :: fv1, fv2 - real(r8) :: rms, min_val(2), max_val(2), avgval(2), m1, rdmax - real(r8) :: rms_normalized ! rms normalized by absolute values - real(r8) :: rms_normalized_denom ! denominator for computing rms_normalized - real(r8) :: rdsum ! sum of relative differences - real(r8) :: rdlogsum ! sum of negative log10 of relative differences - real(r8) :: rdbar ! average of relative differences - real(r8) :: rdlogbar ! rdlogsum normalized by number of non-zero differences - integer :: t(2), n1, n2, min_loc(2), max_loc(2), spacelen - integer :: start(NF90_MAX_DIMS,2), kount(NF90_MAX_DIMS,2), dsizes(NF90_MAX_DIMS,2) - logical, pointer :: mask(:,:) - integer :: diffcnt, rdmaxloc - character(len=80) :: min_str(2), max_str(2), dmax_str, rdmax_str, space - logical :: compare2 - - min_str = '' - max_str = '' - dmax_str = '' - rdmax_str = '' - space = '' - - if(present(tindex)) then - t = tindex - else - t = 1 - end if - - compare2 = (n==2 .and. vid(2)>0) - ifail = 0 - ifilldiff = 0 - idiff = 0 - s1 = vdimsize(file(1)%dim, file(1)%var(vid(1))%dimids) - - if(verbose) print *,__FILE__,__LINE__,s1,file(1)%var(vid(1))%name - - if(compare2) then - s2 = vdimsize(file(2)%dim, file(2)%var(vid(2))%dimids) - - if(s1 /= s2) then - write(6,*) 'WARNING: Variable ',trim(file(1)%var(vid(1))%name),' sizes differ' - write(6,'(a,a32)') ' DIMSIZEDIFF ', file(1)%var(vid(1))%name - ifail = 1 - return - end if - if(file(2)%var(vid(2))%xtype /= file(1)%var(vid(1))%xtype) then - write(6,*) 'WARNING: Variable ',trim(file(1)%var(vid(1))%name),' types differ' - write(6,'(a,a32,2i2)') ' TYPEDIFF ', file(1)%var(vid(1))%name,file(1)%var(vid(1))%xtype,file(2)%var(vid(2))%xtype - itypes = 1 - endif - - end if - n1 = size(file(1)%var(vid(1))%dimids) - - do i=1,n1 - start(i,1) = file(1)%dim(file(1)%var(vid(1))%dimids(i))%start - kount(i,1) = file(1)%dim(file(1)%var(vid(1))%dimids(i))%kount - dsizes(i,1) = file(1)%dim(file(1)%var(vid(1))%dimids(i))%dimsize - if(file(1)%var(vid(1))%dimids(i) == file(1)%unlimdimid) then - start(i,1)=t(1) - dsizes(i,1) = kount(i,1) - end if - end do - - allocate(buf(s1,n)) - - call checknf90(nf90_get_var(file(1)%fh, vid(1), buf(:,1), start(1:n1,1), kount(1:n1,1))) - - - allocate(mask(s1,n)) - ierr = nf90_get_att(file(1)%fh, vid(1), '_FillValue', fv1) - if(ierr == NF90_NOERR) then - mask(:,1) = (buf(:,1)/=fv1) - else - mask(:,1) = .true. - end if - if(n1>0) then - call compute_var_stats(buf(:,1), s2, mask(:,1), min_loc(1), max_loc(1), min_val(1), max_val(1), avgval(1)) - call get_dim_str(n1,translate_loc(n1,min_loc(1),start(1:n1,1),kount(1:n1,1),dsizes(1:n1,1)),min_str(1)) - call get_dim_str(n1,translate_loc(n1,max_loc(1),start(1:n1,1),kount(1:n1,1),dsizes(1:n1,1)),max_str(1)) - end if - space = ' ' - spacelen=1 - if(n1>3) spacelen=(n1-3)*8 ! adjusts the output format - - if(compare2) then - n2 = size(file(2)%var(vid(2))%dimids) - if(n2/=n1) then - print *,'WARNING variable ',trim(file(1)%var(vid(1))%name),& - ' dims differ but total size is the same, will try to compare anyway' - endif - - - do i=1,n2 - start(i,2) = file(2)%dim(file(2)%var(vid(2))%dimids(i))%start - kount(i,2) = file(2)%dim(file(2)%var(vid(2))%dimids(i))%kount - dsizes(i,2) = file(2)%dim(file(2)%var(vid(2))%dimids(i))%dimsize - if(file(2)%var(vid(2))%dimids(i) == file(2)%unlimdimid) then - start(i,2)=t(2) - dsizes(i,2) = kount(i,2) - end if - end do - - call checknf90(nf90_get_var(file(2)%fh, vid(2), buf(:,2), start(1:n2,2), kount(1:n2,2))) - ierr = nf90_get_att(file(2)%fh, vid(2), '_FillValue', fv2) - if(ierr == NF90_NOERR) then - mask(:,2) = (buf(:,2)/=fv2) - else - mask(:,2) = .true. - end if - if(n2>0) then - call compute_var_stats(buf(:,2), s2, mask(:,2), min_loc(2), max_loc(2), min_val(2), max_val(2), avgval(2)) - call get_dim_str(n2,translate_loc(n2,min_loc(2),start(1:n2,2),kount(1:n2,2),dsizes(1:n2,2)),min_str(2)) - call get_dim_str(n2,translate_loc(n2,max_loc(2),start(1:n2,2),kount(1:n2,2),dsizes(1:n2,2)),max_str(2)) - end if - diffcnt=0 - if(any(buf(:,1) /= buf(:,2))) then - allocate(vdiff(s1)) - -! Use the union of mask1 and mask2 - if(any(mask(:,1) .neqv. mask(:,2))) then - write(6,*) 'WARNING: Fill patterns differ between files' - write(6,'(a,a32)') ' FILLDIFF ', file(1)%var(vid(1))%name - ifilldiff = 1 - mask(:,1) = (mask(:,1) .and. mask(:,2)) - end if - - s2 = count(mask(:,1)) - vdiff = abs(buf(:,1)-buf(:,2)) - rms = sqrt(sum(vdiff**2,mask(:,1))/real(s2)) - diffcnt = 0 -#if {ITYPE}==TYPEDOUBLE || {ITYPE}==TYPEREAL - ! Count the NaN values only if they differ between files - do i=1,s1 - if(mask(i,1)) then - if(ieee_is_nan(buf(i,1)) .neqv. ieee_is_nan(buf(i,2))) then - diffcnt = diffcnt + 1 - endif - endif - enddo -#endif - diffcnt = diffcnt + count(vdiff>0 .and. mask(:,1)) - ! Compute normalized rms difference; normalize using the avg abs field - ! values. Note that this differs from the definition of normalized rms - ! difference found in some references (e.g., normalizing by [max - min], which - ! can be sensitive to outliers). - if (n1 > 0 .and. n2 > 0 .and. rms > 0) then - rms_normalized_denom = (avgval(1) + avgval(2)) / 2.0 - if(abs(rms_normalized_denom)>0)then - rms_normalized = rms / rms_normalized_denom - else - rms_normalized = huge(rms) - end if - else - ! don't try to compute rms_normalized in any of the following conditions: - ! n1 = 0 -- then we won't have avgval(1) - ! n2 = 0 -- then we won't have avgval(2) - ! rms = 0 -- then rms_normalized should be 0... but don't try to compute it - ! above in case we have a 0/0 condition - rms_normalized = 0 - end if - - -! diffcnt==0 implies only diffs are in missing values - if(diffcnt>0) then - idiff = 1 - m1 = maxval(vdiff, mask=mask(:,1)) - l1 = maxloc(vdiff, mask=mask(:,1)) - - - if (n1>0) then - call get_dim_str(n1,translate_loc(n1,l1(1),start(1:n1,1),kount(1:n1,1),dsizes(1:n1,1)),dmax_str) - else - dmax_str = ' ' - end if - -#if ({ITYPE} != TYPEINT) - call get_rdiff_stats(s1,buf(:,1),buf(:,2),vdiff,mask(:,1),rdsum, rdlogsum, rdmax, rdmaxloc) - if (n1>0) then - call get_dim_str(n1,translate_loc(n1,rdmaxloc,start(1:n1,1),kount(1:n1,1),dsizes(1:n1,1)),rdmax_str) - else - rdmax_str = ' ' - end if -#endif - - deallocate(vdiff) - - rdbar = rdsum / real(s2) - rdlogbar = rdlogsum / real(diffcnt) - - if(n1==0) then - ! Note that NORMALIZED RMS is NOT computed in this case, so we simply - ! print 0 for that. -#if ({ITYPE} == TYPEINT) - write(6,902) s2, buf(1,1), buf(2,1) - write(6,811) ' RMS ', file(1)%var(vid(1))%name, rms, ' NORMALIZED ', 0 -#else - write(6,803) s2, buf(1,1), buf(2,1) - write(6,812) ' RMS ', file(1)%var(vid(1))%name, rms, ' NORMALIZED ', 0. -#endif - - else - write(6,800) diffcnt, s1, trim(max_str(1)),trim(min_str(1)), trim(dmax_str), trim(rdmax_str) -#if ({ITYPE} == TYPEINT) - ! Note that rdmaxloc is NOT computed in this case, so we print 0 in place - ! of buf(rdmaxloc,1) and buf(rdmaxloc,2) - write(6,903) s2, max_val(1), min_val(1), m1, buf(l1(1),1), rdbar, 0.0, & - count(mask(:,2)), max_val(2), min_val(2), buf(l1(1),2), 0.0 - write(6,810) s1, trim(max_str(2)), trim(min_str(2)) - ! write(6,905) avgval(1), rms, rdbar, avgval(2), rdlogbar, log10(1./rdmax) - write(6,812) ' RMS ', file(1)%var(vid(1))%name, rms, ' NORMALIZED ', rms_normalized -#else - write(6,803) s2, max_val(1), space(1:spacelen),min_val(1), m1, buf(l1(1),1), rdbar, buf(rdmaxloc,1), & - count(mask(:,2)), max_val(2), space(1:spacelen),min_val(2), buf(l1(1),2), buf(rdmaxloc,2) - write(6,810) s1, trim(max_str(2)), trim(min_str(2)) - write(6,805) avgval(1), rms, rdbar, avgval(2), rdlogbar, log10(1./rdmax) - write(6,812) ' RMS ', file(1)%var(vid(1))%name, rms, ' NORMALIZED ', rms_normalized -#endif - endif - endif - end if - if(diffcnt==0) then ! no differences found - if(n1>0) then - write(6,810) s1, trim(max_str(1)),trim(min_str(1)) -#if ({ITYPE} == TYPEINT) - write(6,914) s2, max_val(1), space(1:spacelen),min_val(1), count(mask(:,2)),& - max_val(2),space(1:spacelen),min_val(2) - write(6,810) s1, trim(max_str(2)), trim(min_str(2)) - write(6,815) avgval(1), avgval(2) -#else - write(6,814) s2, max_val(1), space(1:spacelen),min_val(1), count(mask(:,2)),& - max_val(2),space(1:spacelen),min_val(2) - write(6,810) s1, trim(max_str(2)), trim(min_str(2)) - write(6,815) avgval(1), avgval(2) -#endif - endif - end if - else ! Single file analysis output - if(n==2 ) then - write(6,*) 'Variable on file1: ',trim(file(1)%var(vid(1))%name),' not found on file2' - end if - - write(6,810 ) s1, trim(max_str(1)),trim(min_str(1)) - write(6, 825) s2, max_val(1),min_val(1) - write(6, 826) avgval(1) - end if - - deallocate(buf, mask) -800 format(3x,i8,1x,i8,2x,a,1x,a,1x,a,1x,a) -803 format(12x, i8,1x,1pe23.15,a,e23.15,e8.1, e23.15,e8.1,e23.15,/, & - 12x, i8,1x, e23.15,a,e23.15,8x, e23.15,8x, e23.15) - - -805 format(10x,'avg abs field values: ',1pe23.15,4x,'rms diff:',e8.1, & - 3x,'avg rel diff(npos): ',e8.1,/, & - 10x,' ', e23.15,24x, & - 'avg decimal digits(ndif): ',0p,f4.1,' worst: ',f4.1) - -810 format(12x,i8,2x,a,1x,a) - -! RMS for int -811 format(a,a32,1pe11.4,11x,a,i12,/) - -! RMS for real -812 format(a,a32,1pe11.4,11x,a,1pe11.4,/) - -814 format(12x, i8,1x,e23.15,a,e23.15,/, & - 12x, i8,1x,e23.15,a,e23.15) -815 format(10x,'avg abs field values: ',1pe23.15,/, & - 10x,' ', e23.15) -825 format(12x,i8,1x,1p2e23.15) -826 format(12x,'avg abs field values: ',1pe23.15,/) -902 format(12x, i8,1x,i8,a,i8) - -903 format(12x, i8,3e23.15,i8,2e23.15/, & - 12x, i8,2e23.15,23x,i8,23x,e23.15) - - -905 format(10x,'avg abs field values: ',i8,4x,'rms diff:',i8, & - 3x,'avg rel diff(npos): ',i8,/, & - 10x,' ', i8,24x, & - 'avg decimal digits(ndif): ',i8,' worst: ',i8) -914 format(12x, i8,1x,1pe23.15,a,1pe23.15,/, & - 12x, i8,1x,1pe23.15,a,1pe23.15) -915 format(10x,'avg abs field values: ',i8,/, & - 10x,' ', i8) - - end subroutine compare_var_{TYPE} - - ! TYPE real,double - subroutine get_rdiff_stats_{TYPE} (s1, v1, v2, vdiff, mask, rdsum, rdlogsum, rdmax, loc) - integer, intent(in) :: s1 - {VTYPE}, intent(in) :: v1(:), v2(:), vdiff(:) - logical, intent(in) :: mask(:) - real(r8), intent(out) :: rdsum, rdlogsum, rdmax - integer, intent(out) :: loc - real(r8) :: denom, rdiff(s1) - - integer :: i, iloc(1) - - rdiff=0 - rdsum=0 - rdlogsum=0 - do i=1,s1 - if(vdiff(i)>0) then - denom = max(abs(v1(i)), abs(v2(i))) - rdiff(i) = vdiff(i)/denom - rdsum = rdsum+rdiff(i) - rdlogsum = rdlogsum - log10(rdiff(i)) - end if - end do - rdmax = maxval(rdiff) - iloc = maxloc(rdiff) - - loc = iloc(1) - - end subroutine get_rdiff_stats_{TYPE} - - ! TYPE real,int,double - subroutine compute_var_stats_{TYPE} (buf, nvalid, mask, min_loc, max_loc, min_val, max_val, avgval) - {VTYPE}, intent(in) :: buf(:) - logical, intent(in) :: mask(:) - integer, intent(out) :: nvalid, min_loc, max_loc - real(r8), intent(out) :: min_val, max_val, avgval - - integer :: loc(2) - - nvalid = count(mask) - if(nvalid>0) then - loc(1:1) = maxloc(buf, mask=mask) - loc(2:2) = minloc(buf, mask=mask) - max_loc = loc(1) - min_loc = loc(2) - max_val = maxval(buf, mask=mask) - min_val = minval(buf, mask=mask) - avgval = sum(abs(buf),mask=mask)/real(nvalid) - else - max_loc=0 - min_loc=0 - max_val=0 - min_val=0 - avgval=0 - end if - - - - end subroutine compute_var_stats_{TYPE} - - - - - - function translate_loc(ndims, loc, start, kount, dsize) - integer, intent(in) :: ndims, loc, start(:), kount(:), dsize(:) - integer :: translate_loc(ndims) - - integer :: i, tloc, tprod - - tprod = product(kount) - if(loc>tprod) then - write(6,*) 'ERROR in translate_loc: location ',loc,' exceeds array size',tprod - stop - end if - if(ndims<1) then - stop '0D array in translate_loc' - endif - translate_loc = 1 - if(ndims==1) then - translate_loc = loc - else if(loc<=dsize(1)) then - translate_loc(1) = loc - else - tloc = loc - - if(verbose) print *,__LINE__,loc,ndims,dsize(1:ndims) - do i=ndims,1,-1 - tprod = tprod/dsize(i) - if(tloc>=tprod) then - translate_loc(i) = tloc/tprod + start(i) - tloc = tloc - (tloc/tprod)*tprod - end if - end do - translate_loc(1) = translate_loc(1)-1 - - if(verbose) print *,__LINE__,translate_loc(1:ndims) - end if - - end function translate_loc - - - -end module compare_vars_mod diff --git a/CIME/non_py/cprnc/cprnc.F90 b/CIME/non_py/cprnc/cprnc.F90 deleted file mode 100644 index 735b02c6f7d..00000000000 --- a/CIME/non_py/cprnc/cprnc.F90 +++ /dev/null @@ -1,310 +0,0 @@ -program piocprnc - use netcdf - use filestruct - use compare_vars_mod -#ifdef NAGFOR - use f90_unix -#endif - implicit none - - integer :: nargs, n - character(len=1024) :: arg = '' ! cmd-line argument - character(len=1024) :: fname(2) = ' ' ! input filenames - integer :: nchars - integer :: numcases=1 - integer :: ierr -! integer, external :: iargc - type(file_t) :: file(2) - type(dim_t) :: dimoptions(12) - integer :: dimoptioncnt - integer :: nvars, ndiffs, nfilldiffs - - ! The following variables count the number of fields found on one file but not the - ! other, only considering (a) fields with an unlimited (time) dimension, and (b) fields - ! without an unlimited (time) dimension on a file that doesn't have an unlimited - ! dimension. - integer :: num_not_found_on_file1, num_not_found_on_file2 - - ! The following variables count the number of fields found on one file but not the - ! other, only considering fields without an unlimited (time) dimension on a file that - ! has an unlimited dimension. - integer :: num_not_found_on_file1_timeconst, num_not_found_on_file2_timeconst - - integer :: num_sizes_differ - integer :: num_types_differ - integer :: num_not_analyzed -! -! Parse arg list -! - - - nargs = command_argument_count () - dimoptioncnt=0 - ignoretime=.false. - n = 1 - do while (n <= nargs) - arg = ' ' - call getarg (n, arg) - n = n + 1 - select case (arg) - case ('-v') - verbose = .true. - case ('-d') - call getarg(n, arg) - n=n+1 - dimoptioncnt=dimoptioncnt+1 - call parsearg(arg, dimoptions(dimoptioncnt)%name, dimoptions(dimoptioncnt)%start, dimoptions(dimoptioncnt)%kount) - - case ('-m') - ignoretime=.true. - case default - if (fname(1) == ' ') then - fname(1) = arg(1:len_trim(arg)) - nchars = len_trim (fname(1)) - write (6,*) 'file 1=',fname(1)(1:nchars) - else if (fname(2)==' ') then - fname(2) = arg(1:len_trim(arg)) - nchars = len_trim (fname(2)) - write (6,*) 'file 2=',fname(2)(1:nchars) - numcases = 2 - else - call usage_exit (' ') - end if - end select - end do -! -! Must have at least 1 file input -! - if (fname(1) == ' ') then - call usage_exit ('You must enter at least 1 input file') - end if - -! -! Read the files and initialize file_t -! - do n=1, numcases - ierr = nf90_open(fname(n),NF90_NOWRITE, file(n)%fh) - if(ierr /= NF90_NOERR) then - stop 'Failed to open file ' - endif - if(dimoptioncnt>0) then - call init_file_struct( file(n), dimoptions(1:dimoptioncnt) ) - else - call init_file_struct( file(n)) - end if - end do - - if(numcases==2) then - call compare_metadata(file(1), file(2)) - - call compare_dimensions( file(1)%dim, file(2)%dim) - - num_not_found_on_file1 = 0 - num_not_found_on_file2 = 0 - num_not_found_on_file1_timeconst = 0 - num_not_found_on_file2_timeconst = 0 - call match_vars( file(1), file(2), & - num_not_found_on_file1 = num_not_found_on_file1, & - num_not_found_on_file2 = num_not_found_on_file2, & - num_not_found_on_file1_timeconst = num_not_found_on_file1_timeconst, & - num_not_found_on_file2_timeconst = num_not_found_on_file2_timeconst) - end if - call compare_vars(numcases, file, nvars, ndiffs, nfilldiffs, & - num_sizes_differ, num_not_analyzed, num_types_differ) - - -! -! Summarize results -! - write(6,806) - write(6,*) ' ' - write(6,700) 'SUMMARY of cprnc:' - if(numcases==1) then - write(6,700) ' A total number of ',nvars,' fields in file 1 were analyzed (non-compare mode)' - write(6,700) ' A total number of ',num_not_analyzed, & - ' fields in file 1 could not be analyzed' - else - write(6,700) ' A total number of ',nvars,' fields were compared' - write(6,700) ' of which ',ndiffs,' had non-zero differences' - write(6,700) ' and ',nfilldiffs,' had differences in fill patterns' - write(6,700) ' and ',num_sizes_differ,' had different dimension sizes' - write(6,700) ' and ',num_types_differ,' had different data types' - write(6,700) ' A total number of ',num_sizes_differ + num_not_analyzed, & - ' fields could not be analyzed' - - call print_fields_not_found( & - filenum = 1, & - file_has_unlimited_dim = file(1)%has_unlimited_dim(), & - num_not_found = num_not_found_on_file2, & - num_not_found_timeconst = num_not_found_on_file2_timeconst) - - call print_fields_not_found( & - filenum = 2, & - file_has_unlimited_dim = file(2)%has_unlimited_dim(), & - num_not_found = num_not_found_on_file1, & - num_not_found_timeconst = num_not_found_on_file1_timeconst) - - if (nvars == 0 .or. ndiffs > 0 .or. nfilldiffs > 0 .or. & - num_sizes_differ > 0 .or. num_not_analyzed >= nvars .or. & - num_types_differ > 0) then - write(6,700) ' diff_test: the two files seem to be DIFFERENT ' - else if (num_not_found_on_file1 > 0 .or. num_not_found_on_file2 > 0) then - ! Note that we deliberately allow num_not_found_on_file1_timeconst or - ! num_not_found_on_file2_timeconst to be > 0: those do NOT result in a - ! "DIFFER" result. - ! - ! Ideally, we'd count those fields here, too. Doing so would catch more - ! differences and would simplify the cprnc code. But this sometimes leads to - ! problems when comparing restart vs. baseline files - ! (https://github.com/ESMCI/cime/issues/3007). We could add a flag that you - ! specify to not count these fields, but there are backwards compatibility - ! issues with doing so. Eventually it could be good to count these absent - ! fields as a DIFFER result by default, adding a flag that you can specify to - ! not count them, then have cime specify this flag when doing the in-test - ! comparison (so absent time-constant fields would result in a DIFFER result - ! for cime's baseline comparisons and for interactive use of cprnc). - write(6,'(a)') ' diff_test: the two files DIFFER only in their field lists' - else - write(6,700) ' diff_test: the two files seem to be IDENTICAL ' - if (num_not_found_on_file1_timeconst > 0 .or. & - num_not_found_on_file2_timeconst > 0) then - write(6,'(a)') ' (But note that there were differences in field lists just for time-constant fields.)' - end if - end if - end if - write(6,*) ' ' -700 format(a,i6,a) -806 format(132('*')) - - - - contains - subroutine usage_exit (arg) - implicit none - - character(len=*), intent(in) :: arg - - if (arg /= ' ') write (6,*) arg - write(6,*)'Usage: cprnc [-m] [-v] [-d dimname:start[:count]] file1 [file2]' - write(6,*)'-v: Verbose output' - write(6,*)'-m: Ignore time variable and just match contents (default is to match the values in variable time.)' - write(6,*)'-d dimname:start[:count]: Print variable values for the specified dimension index start and count. If not present,' - write(6,*)' count will default to 1. If count is < 0 then count will be set to dimsize-start' - write(6,*)' ' - - stop 999 - end subroutine usage_exit - - - subroutine parsearg (arg, dimname, v1, v2) - !------------------------------------------------------------------------------------------- - ! Purpose: Parse cmd line args about printing. - ! - ! Method: Input is expected in the form: dimname:number1[:number2] where dimname is expected to - ! be the name of a dimension in the input file(s), number1 is the starting position in that - ! dimension to be evaluated and number2 is the number of values to read in the dimension - ! if number2 is missing all remaining values are read. - ! - !------------------------------------------------------------------------------------------- - implicit none - - character(len=*), intent(in) :: arg ! cmd line arg expected of the form 'num1:num2' or 'num1' - - character(len=*), intent(out) :: dimname - integer, intent(out) :: v1 ! e.g. num1 from above example - integer, intent(out) :: v2 ! e.g. num2 from above example - - integer :: i, j ! indices through arg - integer :: ierr ! io error status - - ! - ! First get a dimension name - ! - dimname = ' ' - i = scan(arg,':') - dimname(1:i-1)=arg(1:i-1) - i=i+1 - - ! - ! now try to get an integer number for everything up to ":" - ! - j=i - do while (j < len(arg) .and. arg(j:j) >= '0' .and. arg(j:j) <= '9' .and. arg(j:j) /= ':') - j = j + 1 - end do - read (arg(i:j-1), '(i5)') v1 - ! - ! Next, if ":" comes after the number, look for the next number - ! - i=j - - if (arg(i:i) == ':') then - j = i + 1 - do while (j < len(arg) .and. scan(arg(j:j),"-0123456789")>0) - j = j + 1 - end do - read (arg(i+1:j-1), '(i5)', iostat=ierr) v2 - ! - ! On unexpected input set v2 = -1, e.g. "-d lon:2:blah" will mean get all lons > 1 - ! - if (ierr /= 0) then - v2 = -1 - end if - else - ! - ! ":" not present. Interpret for example '-d lon:2' to mean '-d lon:2:1' - ! - v2 = 1 - end if - if(verbose) print *,__FILE__,__LINE__,trim(dimname),v1,v2 - return - end subroutine parsearg - - subroutine print_fields_not_found(filenum, file_has_unlimited_dim, & - num_not_found, num_not_found_timeconst) - ! Prints information about the number of fields in filenum not found on the other file - - integer, intent(in) :: filenum ! file number for which we're printing this information - logical, intent(in) :: file_has_unlimited_dim ! whether this file has an unlimited dimension - - ! Number of fields in filenum but not on the other file, only considering (a) fields - ! with an unlimited (time) dimension, and (b) fields without an unlimited (time) - ! dimension on a file that doesn't have an unlimited dimension - integer, intent(in) :: num_not_found - - ! Number of fields in filenum but not on the other file, only considering fields - ! without an unlimited (time) dimension on a file that has an unlimited dimension - integer, intent(in) :: num_not_found_timeconst - - integer :: other_filenum - - if (filenum == 1) then - other_filenum = 2 - else if (filenum == 2) then - other_filenum = 1 - else - stop 'Unexpected value for filenum' - end if - - if (file_has_unlimited_dim) then - write(6,'(a,i6,a,i1,a,i1,a)') & - ' A total number of ', num_not_found, & - ' time-varying fields on file ', filenum, & - ' were not found on file ', other_filenum, '.' - write(6,'(a,i6,a,i1,a,i1,a)') & - ' A total number of ', num_not_found_timeconst, & - ' time-constant fields on file ', filenum, & - ' were not found on file ', other_filenum, '.' - else - write(6,'(a,i6,a,i1,a,i1,a)') & - ' A total number of ', num_not_found, & - ' fields on file ', filenum, & - ' were not found on file ', other_filenum, '.' - if (num_not_found_timeconst > 0) then - stop 'Programming error: file has no unlimited dimension, but num_not_found_timeconst > 0' - end if - end if - - end subroutine print_fields_not_found - - end program piocprnc diff --git a/CIME/non_py/cprnc/filestruct.F90 b/CIME/non_py/cprnc/filestruct.F90 deleted file mode 100644 index 814e7faed5e..00000000000 --- a/CIME/non_py/cprnc/filestruct.F90 +++ /dev/null @@ -1,548 +0,0 @@ -module filestruct - use netcdf - implicit none - type dim_t - integer :: dimsize - integer :: start, kount ! used for user requested dimension subsetting - character(len=nf90_MAX_NAME) ::name = '' - end type dim_t - - type var_t - integer :: matchid - integer :: ndims - integer :: natts - integer, pointer :: dimids(:) - integer :: xtype - character(len=nf90_MAX_NAME) ::name = '' - end type var_t - - type file_t - integer :: fh - integer :: natts - type(dim_t), pointer :: dim(:) - type(var_t), pointer :: var(:) - integer :: unlimdimid - contains - procedure :: has_unlimited_dim ! logical function; returns true if this file has an unlimited dimension - end type file_t - - logical :: verbose - -contains - logical function has_unlimited_dim(file) - ! Returns true if this file has an unlimited dimension - class(file_t), intent(in) :: file - - if (file%unlimdimid == -1) then - has_unlimited_dim = .false. - else - has_unlimited_dim = .true. - end if - end function has_unlimited_dim - - subroutine init_file_struct( file, dimoptions ) - - type(file_t) :: file - type(dim_t), optional :: dimoptions(:) - integer :: ndims, nvars - integer :: dimids(NF90_MAX_DIMS) - integer :: i, ierr, docnt, n1, n2 - integer :: j, start, kount - character(len=NF90_MAX_NAME) :: name, dname - ierr= nf90_inquire(file%fh, ndims, nvars, file%natts, file%unlimdimid) - - allocate(file%dim(ndims)) - allocate(file%var(nvars)) - - - do i=1,ndims - ierr = nf90_inquire_dimension(file%fh, i, file%dim(i)%name, file%dim(i)%dimsize) - file%dim(i)%start=1 - if(i==file%unlimdimid) then - file%dim(i)%kount=1 - else - file%dim(i)%kount=file%dim(i)%dimsize - end if - end do - - if(present(dimoptions)) then - docnt = size(dimoptions) - do j=1,docnt - start = dimoptions(j)%start - kount = dimoptions(j)%kount - name = dimoptions(j)%name - n1 = len_trim(name) - do i=1,ndims - dname = file%dim(i)%name - n2 = len_trim(dname) - if(name(1:n1).eq.dname(1:n2) ) then - - - if((start > 0) .and. (start < file%dim(i)%dimsize)) then - file%dim(i)%start = start - else - write(6,*) 'Command line start value for dim ',name(1:n1),& - ' out of bounds, expected 1-',file%dim(i)%dimsize,' got: ',start - stop - end if - if(kount > 0 .and. start+kount <= file%dim(i)%dimsize) then - file%dim(i)%kount = kount - else if(kount == -1) then - file%dim(i)%kount = file%dim(i)%dimsize-file%dim(i)%start+1 - else - write(6,*) 'Command line count value for dim ',name(1:n1),& - ' out of bounds, expected 1-',file%dim(i)%dimsize-file%dim(i)%start+1,' got: ',kount - stop - - endif - write(6,*) 'Setting dimension bounds for dim ',name(1:n1),file%dim(i)%start,file%dim(i)%kount - - exit - end if - end do - end do - end if - - do i=1,nvars - file%var(i)%matchid=-1 - ierr = nf90_inquire_variable(file%fh, i, file%var(i)%name, file%var(i)%xtype, file%var(i)%ndims, dimids, & - file%var(i)%natts) - allocate(file%var(i)%dimids(file%var(i)%ndims)) - file%var(i)%dimids = dimids(1:file%var(i)%ndims) - end do - - - end subroutine init_file_struct - - - subroutine compare_metadata(file1, file2, vid) - type(file_t) :: file1, file2 - integer, optional, intent(in) :: vid - - integer :: id1, id2, natts1, natts2 - - integer :: i, ierr - character(len=NF90_MAX_NAME) :: attname - integer :: atttype, attlen - - real, pointer :: attreal1(:), attreal2(:) - double precision, pointer :: attdouble1(:),attdouble2(:) - integer, pointer :: attint1(:),attint2(:) - integer, parameter :: maxstrlen=32767 - character(len=maxstrlen) :: attchar1, attchar2 - logical :: found - - - if(present(vid)) then - id1 = vid - id2 = file1%var(id1)%matchid - ierr = nf90_inquire_variable(file1%fh, id1, nAtts=natts1) - ierr = nf90_inquire_variable(file2%fh, id2, nAtts=natts2) - else - id1 = NF90_GLOBAL - id2 = NF90_GLOBAL - natts1 = file1%natts - natts2 = file2%natts - end if - - do i=1,natts1 - found = .true. - attname = '' - ierr = nf90_inq_attname(file1%fh, id1, i, attname) - ierr = nf90_inquire_attribute(file1%fh, id1, trim(attname), atttype, attlen) - - select case(atttype) - case(nf90_char) - if (attlen > maxstrlen) then - stop 'maximum string length exceeded' - endif - attchar1=' ' - attchar2=' ' - - ierr = nf90_get_att(file1%fh,id1, trim(attname), attchar1) - ierr = nf90_get_att(file2%fh,id2, trim(attname), attchar2) - if(ierr==NF90_NOERR) then - if(trim(attname).ne.'case' .and. attchar1(1:attlen) .ne. attchar2(1:attlen)) then - print *, 'Attribute ',trim(attname),' from file1: ',attchar1(1:attlen),& - ' does not match that found on file2: ',attchar2(1:attlen) - end if - else - print *, 'Attribute ',trim(attname),' from file1: ',attchar1(1:attlen),& - ' not found on file2' - end if - if(id1==NF90_GLOBAL .and. trim(attname) .eq. 'case') then - print *, 'CASE 1 : ',trim(attchar1) - print *, 'CASE 2 : ',trim(attchar2) - endif - if(id1==NF90_GLOBAL .and. trim(attname) .eq. 'title') then - print *, 'TITLE 1 : ',trim(attchar1) - print *, 'TITLE 2 : ',trim(attchar2) - end if - case(nf90_int) - allocate(attint1(attlen),attint2(attlen)) - ierr = nf90_get_att(file1%fh,id1, trim(attname), attint1) - ierr = nf90_get_att(file2%fh,id2, trim(attname), attint2) - - if(ierr==NF90_NOERR) then - if(any(attint1 /= attint2)) then - print *, 'Attribute ',trim(attname),' from file1: ',attint1,' does not match that found on file2 ',attint2 - end if - else - print *, 'Attribute ',trim(attname),' from file1: ',attint1,' not found on file2' - end if - deallocate(attint1, attint2) - - - case(nf90_float) - allocate(attreal1(attlen),attreal2(attlen)) - ierr = nf90_get_att(file1%fh,id1, trim(attname), attreal1) - ierr = nf90_get_att(file2%fh,id2, trim(attname), attreal2) - if(ierr==NF90_NOERR) then - if(any(attreal1 /= attreal2)) then - print *, 'Attribute ',trim(attname),' from file1: ',attreal1,' does not match that found on file2 ',attreal2 - end if - else - print *, 'Attribute ',trim(attname),' from file1: ',attreal1,' not found on file2' - end if - deallocate(attreal1, attreal2) - case(nf90_double) - allocate(attdouble1(attlen), attdouble2(attlen)) - ierr = nf90_get_att(file1%fh,id1, trim(attname), attdouble1) - ierr = nf90_get_att(file2%fh,id2, trim(attname), attdouble2) - if(ierr==NF90_NOERR) then - if(any(attdouble1 /= attdouble2)) then - print *, 'Attribute ',trim(attname),' from file1: ',attdouble1,' does not match that found on file2 ',attdouble2 - end if - else - print *, 'Attribute ',trim(attname),' from file1: ',attdouble1,' not found on file2' - end if - deallocate(attdouble1, attdouble2) - case default - print *,' Did not recognize attribute with id: ',i,' type: ',atttype, ' name: ',trim(attname), ' len: ',attlen - end select - end do - - end subroutine compare_metadata - - - - - - - - - subroutine compare_dimensions( dimfile1, dimfile2) - type(dim_t), intent(in) :: dimfile1(:), dimfile2(:) - - integer :: ds1, ds2 - integer :: i, j - logical,pointer :: found(:,:) - - ds1 = size(dimfile1) - ds2 = size(dimfile2) - - allocate(found(2,max(ds1,ds2))) - - found = .false. - do i=1,ds1 - do j=1,ds2 - if(dimfile1(i)%name .eq. dimfile2(j)%name) then - if(dimfile1(i)%dimsize == dimfile2(j)%dimsize) then - print *, 'Dimension ',trim(dimfile1(i)%name), ' matches' - else - print *, 'Dimension ',trim(dimfile1(i)%name), ' differs ', dimfile1(i)%dimsize, ' /= ',dimfile2(j)%dimsize - end if - found(1,i) = .true. - found(2,j) = .true. - end if - end do - end do - do i=1,ds1 - if(.not. found(1,i)) then - print *, 'Could not find match for file 1 dimension ',trim(dimfile1(i)%name) - end if - end do - do i=1,ds2 - if(.not. found(2,i)) then - print *, 'Could not find match for file 2 dimension ',trim(dimfile2(i)%name) - end if - end do - deallocate(found) - end subroutine compare_dimensions - - - subroutine match_vars( file1, file2, & - num_not_found_on_file1, num_not_found_on_file2, & - num_not_found_on_file1_timeconst, num_not_found_on_file2_timeconst) - type(file_t), intent(inout) :: file1, file2 - - ! Accumulates count of variables on file2 not found on file1; this only considers (a) - ! fields with an unlimited (time) dimension, and (b) fields without an unlimited - ! (time) dimension on a file that doesn't have an unlimited dimension. - integer, intent(inout) :: num_not_found_on_file1 - - ! Accumulates count of variables on file1 not found on file2; this only considers (a) - ! fields with an unlimited (time) dimension, and (b) fields without an unlimited - ! (time) dimension on a file that doesn't have an unlimited dimension. - integer, intent(inout) :: num_not_found_on_file2 - - ! Accumulates count of variables on file2 not found on file1; this only considers - ! fields without an unlimited (time) dimension on a file that has an unlimited - ! dimension. - integer, intent(inout) :: num_not_found_on_file1_timeconst - - ! Accumulates count of variables on file1 not found on file2; this only considers - ! fields without an unlimited (time) dimension on a file that has an unlimited - ! dimension. - integer, intent(inout) :: num_not_found_on_file2_timeconst - - type(var_t), pointer :: varfile1(:),varfile2(:) - - integer :: vs1, vs2, i, j - - - - varfile1 => file1%var - varfile2 => file2%var - - vs1 = size(varfile1) - vs2 = size(varfile2) - - do i=1,vs1 - do j=1,vs2 - if(varfile1(i)%name .eq. varfile2(j)%name) then - varfile1(i)%matchid=j - varfile2(j)%matchid=i - end if - end do - end do - do i=1,vs1 - if(varfile1(i)%matchid<0) then - print *, 'Could not find match for file1 variable ',trim(varfile1(i)%name), ' in file2' - if (file1%has_unlimited_dim() .and. & - .not. is_time_varying(varfile1(i), file1%has_unlimited_dim(), file1%unlimdimid)) then - num_not_found_on_file2_timeconst = num_not_found_on_file2_timeconst + 1 - else - num_not_found_on_file2 = num_not_found_on_file2 + 1 - end if - end if - end do - do i=1,vs2 - if(varfile2(i)%matchid<0) then - print *, 'Could not find match for file2 variable ',trim(varfile2(i)%name), ' in file1' - if (file2%has_unlimited_dim() .and. & - .not. is_time_varying(varfile2(i), file2%has_unlimited_dim(), file2%unlimdimid)) then - num_not_found_on_file1_timeconst = num_not_found_on_file1_timeconst + 1 - else - num_not_found_on_file1 = num_not_found_on_file1 + 1 - end if - end if - end do - end subroutine match_vars - - - function is_time_varying(var, file_has_unlimited_dim, unlimdimid) - type(var_t), intent(in) :: var ! variable of interest - logical , intent(in) :: file_has_unlimited_dim ! true if the file has an unlimited dimension - integer , intent(in) :: unlimdimid ! the file's unlimited dim id (if it has one) - - logical :: is_time_varying ! true if the given variable is time-varying - - if (file_has_unlimited_dim) then - is_time_varying = any(var%dimids == unlimdimid) - else - is_time_varying = .false. - end if - end function is_time_varying - - - function vdimsize(dims, dimids) - type(dim_t), intent(in) :: dims(:) - integer, intent(in) :: dimids(:) - - integer :: vdimsize - integer :: i - - vdimsize=1 - do i=1,size(dimids) - if(verbose) print *,__FILE__,__LINE__,i,dimids(i),size(dims),size(dimids) - vdimsize = vdimsize*dims(dimids(i))%kount - end do - - end function vdimsize - - - - - - subroutine compare_var_int(f1, f2, i1, i2, t) - type(file_t) :: f1,f2 - integer, intent(in) :: i1, i2 - integer, optional :: t - - - integer :: s1, s2, m1, m2, l1(1), l2(1), i, ierr - integer, pointer :: v1(:), v2(:), vdiff(:) - integer :: t1, n1 - integer :: start(NF90_MAX_DIMS), count(NF90_MAX_DIMS) - - if(present(t)) then - t1 = t - else - t1 = 1 - end if - - s1 = vdimsize(f1%dim, f1%var(i1)%dimids) - s2 = vdimsize(f2%dim, f2%var(i2)%dimids) - - if(s1 /= s2) then - print *, 'Variable ',f1%var(i)%name,' sizes differ' - end if - - n1 = size(f1%var(i1)%dimids) - start = 1 - do i=1,n1 - count(i) = f1%dim(f1%var(i1)%dimids(i))%dimsize - if(f1%var(i1)%dimids(i) == f1%unlimdimid) then - count(i)=1 - start(i)=t1 - end if - end do - - allocate(v1(s1), v2(s2)) - - ierr = nf90_get_var(f1%fh, i1, v1, start(1:n1), count(1:n1)) - ierr = nf90_get_var(f2%fh, i2, v2, start(1:n1), count(1:n1)) - - if(any(v1 /= v2)) then - allocate(vdiff(s1)) - vdiff = abs(v1-v2) - m1 = maxval(vdiff) - m2 = minval(vdiff) - l1 = maxloc(vdiff) - l2 = minloc(vdiff) - - print *,__FILE__,__LINE__,m1,m2,l1,l2 - deallocate(vdiff) - end if - - deallocate(v1,v2) - end subroutine compare_var_int - - subroutine compare_var_float(f1, f2, i1, i2, t) - type(file_t) :: f1,f2 - integer, intent(in) :: i1, i2 - integer, optional :: t - - - integer :: s1, s2, m1, m2, l1(1), l2(1), i, ierr - real, pointer :: v1(:), v2(:), vdiff(:) - integer :: t1, n1 - integer :: start(NF90_MAX_DIMS), count(NF90_MAX_DIMS) - - if(present(t)) then - t1 = t - else - t1 = 1 - end if - - s1 = vdimsize(f1%dim, f1%var(i1)%dimids) - s2 = vdimsize(f2%dim, f2%var(i2)%dimids) - - if(s1 /= s2) then - print *, 'Variable ',f1%var(i)%name,' sizes differ' - end if - - n1 = size(f1%var(i1)%dimids) - start = 1 - do i=1,n1 - count(i) = f1%dim(f1%var(i1)%dimids(i))%dimsize - if(f1%var(i1)%dimids(i) == f1%unlimdimid) then - count(i)=1 - start(i)=t1 - end if - end do - - allocate(v1(s1), v2(s2)) - - ierr = nf90_get_var(f1%fh, i1, v1, start(1:n1), count(1:n1)) - ierr = nf90_get_var(f2%fh, i2, v2, start(1:n1), count(1:n1)) - - if(any(v1 /= v2)) then - allocate(vdiff(s1)) - vdiff = abs(v1-v2) - m1 = maxval(vdiff) - m2 = minval(vdiff) - l1 = maxloc(vdiff) - l2 = minloc(vdiff) - - print *,__FILE__,__LINE__,m1,m2,l1,l2 - deallocate(vdiff) - end if - - deallocate(v1,v2) - end subroutine compare_var_float - - subroutine compare_var_double(f1, f2, i1, i2, t) - type(file_t) :: f1,f2 - integer, intent(in) :: i1, i2 - integer, optional :: t - - - integer :: s1, s2, m1, m2, l1(1), l2(1), i, ierr - double precision, pointer :: v1(:), v2(:), vdiff(:) - integer :: t1, n1 - integer :: start(NF90_MAX_DIMS), count(NF90_MAX_DIMS) - - if(present(t)) then - t1 = t - else - t1 = 1 - end if - - s1 = vdimsize(f1%dim, f1%var(i1)%dimids) - s2 = vdimsize(f2%dim, f2%var(i2)%dimids) - - if(s1 /= s2) then - print *, 'Variable ',f1%var(i)%name,' sizes differ' - end if - - n1 = size(f1%var(i1)%dimids) - start = 1 - do i=1,n1 - count(i) = f1%dim(f1%var(i1)%dimids(i))%dimsize - if(f1%var(i1)%dimids(i) == f1%unlimdimid) then - count(i)=1 - start(i)=t1 - end if - end do - - allocate(v1(s1), v2(s2)) - - ierr = nf90_get_var(f1%fh, i1, v1, start(1:n1), count(1:n1)) - ierr = nf90_get_var(f2%fh, i2, v2, start(1:n1), count(1:n1)) - - if(any(v1 /= v2)) then - allocate(vdiff(s1)) - vdiff = abs(v1-v2) - m1 = maxval(vdiff) - m2 = minval(vdiff) - l1 = maxloc(vdiff) - l2 = minloc(vdiff) - - print *,__FILE__,__LINE__,m1,m2,l1,l2 - deallocate(vdiff) - end if - - deallocate(v1,v2) - end subroutine compare_var_double - - - - - - - - - -end module filestruct diff --git a/CIME/non_py/cprnc/prec.F90 b/CIME/non_py/cprnc/prec.F90 deleted file mode 100644 index abbaba26ea7..00000000000 --- a/CIME/non_py/cprnc/prec.F90 +++ /dev/null @@ -1,8 +0,0 @@ -module prec -! -! Constants for setting precision -! - integer, parameter :: r4 = selected_real_kind (6) - integer, parameter :: r8 = selected_real_kind (12) - integer, parameter :: i4 = selected_int_kind (6) -end module prec diff --git a/CIME/non_py/cprnc/run_tests b/CIME/non_py/cprnc/run_tests deleted file mode 100755 index 98d1a22f3bd..00000000000 --- a/CIME/non_py/cprnc/run_tests +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env perl -# -# Run all cprnc tests -# See usage message for details -# -# Bill Sacks -# 5-28-13 - -use strict; -use Getopt::Long; - -#---------------------------------------------------------------------- -# Define parameters -#---------------------------------------------------------------------- - -# Hash giving info about each test. Key is the test file; value is a -# hash reference containing at least the associated control file (key: -# control), and possibly extra arguments to cprnc (key: extra_args) -my %tests = ('copy.nc' => {control => 'control.nc'}, - 'extra_variables.nc' => {control => 'control.nc'}, - 'diffs_in_vals.nc' => {control => 'control.nc'}, - 'diffs_in_vals_and_extra_and_missing.nc' => {control => 'control.nc'}, - 'diffs_in_fill.nc' => {control => 'control.nc'}, - 'diffs_in_vals_and_diffs_in_fill.nc' => {control => 'control.nc'}, - 'diffs_in_vals_and_fill.nc' => {control => 'control.nc'}, - 'lon_differs.nc' => {control => 'control.nc'}, - 'missing_variables.nc' => {control => 'control.nc'}, - 'vals_differ_by_1.1.nc' => {control => 'control.nc'}, - 'vals_differ_by_1.1_somewhere.nc' => {control => 'control.nc'}, - 'vals_differ_by_varying_amounts.nc' => {control => 'control.nc'}, - 'vals_differ_by_varying_amounts2.nc' => {control => 'control.nc'}, - - 'int_missing.nc' => {control => 'control_int.nc'}, - - 'multipleTimes_someTimeless_diffs_in_vals_and_fill.nc' => {control => 'control_multipleTimes_someTimeless.nc'}, - 'multipleTimes_someTimeless_extra_and_missing.nc' => {control => 'control_multipleTimes_someTimeless.nc'}, - - 'noTime_diffs_in_vals_and_fill.nc' => {control => 'control_noTime.nc', - extra_args => '-m'}, - 'noTime_extra_and_missing.nc' => {control => 'control_noTime.nc', - extra_args => '-m'}, - - 'diffs_0d.nc' => {control => 'control_0d.nc', - extra_args => '-m'}, - - 'cpl.hi.subset.test.nc' => {control => 'cpl.hi.subset.control.nc'}, - 'clm2.h0.subset.test.nc' => {control => 'clm2.h0.subset.control.nc'}, - 'clm2.h1.subset.test.nc' => {control => 'clm2.h1.subset.control.nc'}, - - 'diffs_in_attribute.nc' => {control => 'control_attributes.nc'}, - - 'copy_char.nc' => {control => 'control_char.nc', - extra_args => '-m'}, - - 'diffs_in_nans.nc' => {control => 'control_floatDoubleNan.nc'}, - ); - -#---------------------------------------------------------------------- -# Get arguments and check them -#---------------------------------------------------------------------- - -my %opts; -GetOptions( - "outdir=s" => \$opts{'outdir'}, - "h|help" => \$opts{'help'}, -) or usage(); - -usage() if $opts{'help'}; - -if (@ARGV) { - print "ERROR: unrecognized arguments: @ARGV\n"; - usage(); -} - -if (!$opts{'outdir'}) { - print "ERROR: -outdir must be provided\n"; - usage(); -} - - -#---------------------------------------------------------------------- -# Main script -#---------------------------------------------------------------------- - -mkdir $opts{'outdir'} or die "ERROR creating directory $opts{'outdir'}; -note that this directory should NOT exist before running this script\n"; - -my $num_tests = keys %tests; -print "Running $num_tests tests...\n"; - -foreach my $test (keys %tests) { - print "$test\n"; - my $test_file = $test; - my $control_file = $tests{$test}{'control'}; - my $outfile = "$opts{'outdir'}/${test}.out"; - - my $extra_args = $tests{$test}{'extra_args'}; - - open (my $file, ">", "$outfile") or die "ERROR opening $outfile"; - print $file `./cprnc $extra_args test_inputs/$control_file test_inputs/$test_file`; - close $file; -} - -#---------------------------------------------------------------------- -# Subroutines -#---------------------------------------------------------------------- - -sub usage { - die < [OPTIONS] - - Run all cprnc tests, putting output in directory given by . - should NOT exist before running this script. - - -OPTIONS - -help [or -h] Display this help - -EOF -} diff --git a/CIME/non_py/cprnc/summarize_cprnc_diffs b/CIME/non_py/cprnc/summarize_cprnc_diffs deleted file mode 100755 index 84933192cc6..00000000000 --- a/CIME/non_py/cprnc/summarize_cprnc_diffs +++ /dev/null @@ -1,573 +0,0 @@ -#!/usr/bin/env perl -# -# Summarize cprnc output from all tests in a CESM test suite. -# See usage message for details. -# -# Bill Sacks -# 4-10-13 - -use strict; -use Getopt::Long; -use File::Basename; -use List::Util qw(max sum); - -#---------------------------------------------------------------------- -# Get arguments and check them -#---------------------------------------------------------------------- - -my %opts; -# set defaults -$opts{'output_suffix'} = ''; -# set up options -GetOptions( - "basedir=s" => \$opts{'basedir'}, - "testid=s" => \$opts{'testid'}, - "output_suffix=s" => \$opts{'output_suffix'}, - "narrow" => \$opts{'narrow'}, - "h|help" => \$opts{'help'}, -) or usage(); - -usage() if $opts{'help'}; - -if (@ARGV) { - print "ERROR: unrecognized arguments: @ARGV\n"; - usage(); -} - -if (!$opts{'basedir'}) { - print "ERROR: -basedir must be provided\n"; - usage(); -} -if (!$opts{'testid'}) { - print "ERROR: -testid must be provided\n"; - usage(); -} - -#---------------------------------------------------------------------- -# Main script -#---------------------------------------------------------------------- - -# Create hash containing summary of cprnc differences. This is a reference to a hash, with: -# Keys: "Directory Filename Variable" -# Values: Reference to a hash containing: -# Dir => directory (gives test name) -# Filename => cprnc filename -# Variable => variable -# RMS => rms value [may or may not be present] -# RMS_NORM => normalized rms value [may or may not be present] -# FILLDIFF => ' ' [may or may not be present] -# DIMSIZEDIFF => ' ' [may or may not be present] -my ($summary_hash) = - process_cprnc_output($opts{'basedir'}, $opts{'testid'}, $opts{'output_suffix'}); - -my $outbase="cprnc.summary.$opts{'testid'}"; -if ($opts{'output_suffix'}) { - $outbase = "$outbase.$opts{'output_suffix'}"; -} - -# set widths of output strings -my $widths_hash; -if ($opts{'narrow'}) { - $widths_hash = { Dir => 40, Filename => 40, Variable => 40 }; -} -else { - $widths_hash = max_widths($summary_hash); -} - - -print_results_by_test("${outbase}.by_test", $summary_hash, $widths_hash); -print_results_by_varname("${outbase}.by_varname", $summary_hash, $widths_hash); -print_results_by_rms("${outbase}.by_rms", $summary_hash, $widths_hash); - - - -#---------------------------------------------------------------------- -# Subroutines -#---------------------------------------------------------------------- - -sub usage { - die < -testid [OPTIONS] - - is the base directory in which test directories can be found - - is the testid of the tests to summarize - (can contain shell wildcards) - - This script can be used to post-process and summarize baseline comparison - output from one or more CESM test suites. - - The script finds all directories in basedir whose name ends with the given - testid; these are the test directories of interest. It then examines the - 'run' subdirectory of each test directory of interest, looking for files of - the form *.nc.cprnc.out. Or, if the -output_suffix argument is given, then - it looks for files of the form *.nc.cprnc.out.SUFFIX. (With this naming - convention [i.e., looking for files of the form *.nc.cprnc.out], note that - it only looks at output for baseline comparisons - NOT output from the test - itself, such as cprnc output files from the exact restart test.) (Actually, - we also allow for files of the form *.nc_[0-9][0-9][0-9][0-9].cprnc.out, - such as *.nc_0001.cprnc.out and *.nc_0002.cprnc.out, to pick up - multi-instance files.) - - Summaries of cprnc differences (RMS and normalized RMS differences, FILLDIFFs and DIMSIZEDIFFs) - are placed in three output files beginning with the name 'cprnc.summary', in - the current directory. These files contain the same information, but one is - sorted by test name, one is sorted by variable name, and is one sorted from - largest to smallest normalized RMS differences. - - -OPTIONS - -output_suffix If provided, look for files of the form *.nc.cprnc.out.SUFFIX - rather than just *.nc.cprnc.out - - -narrow Use generally-narrower output field widths to aid readability, - at the expense of truncated strings - - -help [or -h] Display this help - -EOF -} - - -# process_cprnc_output -# Read through all cprnc files, and build hashes of instances of RMS, normalized RMS, FILLDIFF and DIMSIZEDIFF -# Inputs: -# - basedir -# - testid -# - output_suffix -# Output: hash reference -# Dies with an error if no cprnc output files are found -sub process_cprnc_output { - my ($basedir, $testid, $output_suffix) = @_; - - my %diffs; - my $num_files = 0; - - my @test_dirs = glob "${basedir}/*${testid}"; - - foreach my $test_dir (@test_dirs) { - my $test_dir_base = basename($test_dir); - - my @cprnc_files; - if ($output_suffix) { - @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out.${output_suffix} ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out.${output_suffix}"; - } - else { - @cprnc_files = glob "${test_dir}/run/*.nc.cprnc.out ${test_dir}/run/*.nc_[0-9][0-9][0-9][0-9].cprnc.out"; - } - - foreach my $cprnc_file (@cprnc_files) { - my $cprnc_file_base = basename($cprnc_file); - $num_files++; - - open IN, "<", $cprnc_file or die "ERROR opening ${cprnc_file}"; - - while (my $line = ) { - chomp $line; - - process_line($line, $test_dir_base, $cprnc_file_base, \%diffs); - } # while - - close IN; - } # foreach cprnc_file - - } # foreach test_dir - - if ($num_files == 0) { - die "ERROR: no cprnc.out files found\n"; - } - - return \%diffs; -} - - -# process_line: Process one line from one file -# Inputs: -# - line -# - test_dir -# - cprnc_file -# - diffs hash reference (MODIFIED) -sub process_line { - my ($line, $test_dir, $cprnc_file, $diffs) = @_; - - my $diff_type; - my $varname; - my $rms; - my $ignore; - my $rms_normalized; - - if ($line =~ /^ *RMS /) { - ($diff_type, $varname, $rms, $ignore, $rms_normalized) = split " ", $line; - } elsif ($line =~ /^ *FILLDIFF /) { - ($diff_type, $varname) = split " ", $line; - $rms = ""; - $rms_normalized = ""; - } elsif ($line =~ /^ *DIMSIZEDIFF /) { - ($diff_type, $varname) = split " ", $line; - $rms = ""; - $rms_normalized = ""; - } else { - $diff_type = ""; - } - - if ($diff_type eq 'RMS' || $diff_type eq 'FILLDIFF' || $diff_type eq 'DIMSIZEDIFF') { - # We have found a cprnc difference - - my $key = "$test_dir $cprnc_file $varname"; - - # For RMS errors, keep the highest error found - if ($diff_type eq "RMS") { - if (exists $diffs->{$key} && exists $diffs->{$key}{'RMS_NORM'}) { - if ($diffs->{$key}{'RMS_NORM'} > $rms_normalized) { - warn "WARNING: Ignoring lower RMS value: $key : $rms_normalized < $diffs->{$key}{'RMS_NORM'}\n"; - return; - } - else { - warn "WARNING: Replacing RMS with higher value: $key : $rms_normalized > $diffs->{$key}{'RMS_NORM'}\n"; - } - } - } - - # If the diffs hash doesn't already contain information about this - # directory/filename/variable combo, then we need to create a hash - # reference with the appropriate basic metadata. - if (!exists $diffs->{$key}) { - $diffs->{$key} = { - Dir => $test_dir, - Filename => $cprnc_file, - Variable => $varname, - }; - } - - # Whether or not the hash already contained the given key, we need to add - # the value of interest -- either the RMS and normalized RMS errors, or - # the fact that there is a FILLDIFF or DIMSIZEDIFF. - if ($diff_type eq "RMS") { - $diffs->{$key}{'RMS'} = $rms; - $diffs->{$key}{'RMS_NORM'} = $rms_normalized; - } else { - # No meaningful value here - just record the fact that we saw a - # FILLDIFF or DIMSIZEDIFF - $diffs->{$key}{$diff_type} = ""; - } - } elsif ($diff_type ne '') { - die "Unexpected diff_type: $diff_type"; - } -} - - -# max_widths -# Inputs: -# - summary_hash (hash reference) -# Output: reference to a hash containing the maximum width of each of -# the following in the summary hash: -# - Dir -# - Filename -# - Variable -sub max_widths { - my $summary_hash = shift; - - my %maxes; - - foreach my $var ('Dir','Filename','Variable') { - $maxes{$var} = max (map { length($summary_hash->{$_}{$var}) } keys %$summary_hash); - } - - return \%maxes; -} - - -# print_results_by_test: Print sorted hash entries to a file, sorted by test name -# Inputs: -# - outfile: name of output file -# - summary_hash: hash reference containing results to print -# - widths: hash reference giving widths of output strings -sub print_results_by_test { - my ($outfile, $summary_hash, $widths) = @_; - - open OUT, ">", "$outfile" or die "ERROR opening $outfile"; - - my @sorted_keys = sort{ $summary_hash->{$a}{'Dir'} cmp $summary_hash->{$b}{'Dir'} - or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'} - or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} } - keys %$summary_hash; - - my $last_dir; - my $last_filename; - - my $separator_width = sum(values %$widths) + 57; - - for my $key (@sorted_keys) { - - # Print a separator line between different files - if ($summary_hash->{$key}{'Dir'} ne $last_dir || - $summary_hash->{$key}{'Filename'} ne $last_filename) { - if ($last_dir && $last_filename) { - print OUT "=" x $separator_width . "\n"; - } - $last_dir = $summary_hash->{$key}{'Dir'}; - $last_filename = $summary_hash->{$key}{'Filename'}; - } - - my $line = format_line($summary_hash->{$key}, $widths); - - print OUT "$line\n"; - } - - close OUT; -} - - -# print_results_by_varname: Print sorted hash entries to a file, sorted by variable name -# Inputs: -# - outfile: name of output file -# - summary_hash: hash reference containing results to print -# - widths: hash reference giving widths of output strings -sub print_results_by_varname { - my ($outfile, $summary_hash, $widths) = @_; - - open OUT, ">", "$outfile" or die "ERROR opening $outfile"; - - my @sorted_keys = sort{ $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} - or $summary_hash->{$a}{'Dir'} cmp $summary_hash->{$b}{'Dir'} - or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'} } - keys %$summary_hash; - - my $last_variable; - - my $separator_width = sum(values %$widths) + 57; - - for my $key (@sorted_keys) { - - # Print a separator line between different variables - if ($summary_hash->{$key}{'Variable'} ne $last_variable) { - if ($last_variable) { - print OUT "=" x $separator_width . "\n"; - } - $last_variable = $summary_hash->{$key}{'Variable'}; - } - - my $line = format_line($summary_hash->{$key}, $widths); - - print OUT "$line\n"; - } - - close OUT; -} - - - -# print_results_by_rms: Print sorted hash entries to a file, sorted by RMS_NORM -# Inputs: -# - outfile: name of output file -# - summary_hash: hash reference containing results to print -# - widths: hash reference giving widths of output strings -sub print_results_by_rms { - my ($outfile, $summary_hash, $widths) = @_; - - open OUT, ">", "$outfile" or die "ERROR opening $outfile"; - - my @sorted_keys = sort {$summary_hash->{$b}{'RMS_NORM'} <=> $summary_hash->{$a}{'RMS_NORM'} - or $summary_hash->{$a}{'Dir'} cmp $summary_hash->{$b}{'Dir'} - or $summary_hash->{$a}{'Filename'} cmp $summary_hash->{$b}{'Filename'} - or $summary_hash->{$a}{'Variable'} cmp $summary_hash->{$b}{'Variable'} } - keys %$summary_hash; - - for my $key (@sorted_keys) { - my $line = format_line($summary_hash->{$key}, $widths); - - print OUT "$line\n"; - } - - close OUT; -} - - -# Inputs: -# - reference to a hash containing: -# - Dir -# - Filename -# - Variable -# - RMS (optional) -# - RMS_NORM (optional) -# - FILLDIFF (optional) -# - DIMSIZEDIFF (optional) -# - widths: hash reference giving widths of output strings -# Return a formatted line for printing -sub format_line { - my ($hash_ref, $widths) = @_; - - my $dir = $hash_ref->{'Dir'}; - my $filename = $hash_ref->{'Filename'}; - my $variable = $hash_ref->{'Variable'}; - my $rms = ""; - my $rms_normalized = ""; - my $filldiff = ""; - my $dimsizediff = ""; - if (exists $hash_ref->{'RMS'}) { - $rms = sprintf(" : RMS %-16g", $hash_ref->{'RMS'}); - } - if (exists $hash_ref->{'RMS_NORM'}) { - $rms_normalized = sprintf(" : RMS_NORM %-16g", $hash_ref->{'RMS_NORM'}); - } - if (exists $hash_ref->{'FILLDIFF'}) { - $filldiff = " : FILLDIFF"; - } - if (exists $hash_ref->{'DIMSIZEDIFF'}) { - $dimsizediff = " : DIMSIZEDIFF"; - } - - # for width=40, the format string will contain '%-40.40s' - my $format = '%-' . $widths->{'Dir'} . '.' . $widths->{'Dir'} . 's : ' . - '%-' . $widths->{'Filename'} . '.' . $widths->{'Filename'} . 's : ' . - '%-' . $widths->{'Variable'} . '.' . $widths->{'Variable'} . 's' . - '%s%s%s%s'; - - sprintf($format, $dir, $filename, $variable, $filldiff, $dimsizediff, $rms, $rms_normalized); -} - -#======================================================================= -# Notes about testing: unit tests -#======================================================================= - -#----------------------------------------------------------------------- -# Testing process_line -#----------------------------------------------------------------------- - -# use Data::Dumper; - -# my %diffs; - -# # shouldn't do anything -# process_line("hello", "test_dir1", "file_a", \%diffs); - -# # test basic filldiff -# process_line("FILLDIFF var1", "test_dir1", "file_b", \%diffs); - -# # add an RMS to existing filldiff -# process_line("RMS var1 4200 NORMALIZED 42", "test_dir1", "file_b", \%diffs); - -# # test basic rms error -# process_line("RMS var17 0.314 NORMALIZED 3.14", "test_dir1", "file_b", \%diffs); - -# # add a filldiff to existing rms error -# process_line("FILLDIFF var17", "test_dir1", "file_b", \%diffs); - -# # add a filldiff without RMS -# process_line("FILLDIFF var42", "test_dir2", "file_c", \%diffs); - -# # add a dimsizediff -# process_line("DIMSIZEDIFF var43", "test_dir2", "file_c", \%diffs); - -# # add an RMS error without filldiff -# process_line("RMS var100 99 NORMALIZED 100", "test_dir2", "file_d", \%diffs); - -# # test a warning: should issue a warning and replace the above setting -# process_line("RMS var100 9 NORMALIZED 200", "test_dir2", "file_d", \%diffs); - -# # test a warning: should issue a warning but NOT replace the above setting -# # (normalized RMS is smaller even though standard RMS is bigger: the normalized -# # one should be considered in deciding whether to replace the previous setting) -# process_line("RMS var100 999 NORMALIZED 50", "test_dir2", "file_d", \%diffs); - -# print Dumper(\%diffs); - - -# THE ABOVE SHOULD PRINT SOMETHING LIKE THIS (though the output from Dumper will -# likely appear in a different order): - -# WARNING: Replacing RMS with higher value: test_dir2 file_d var100 : 200 > 100 -# WARNING: Ignoring lower RMS value: test_dir2 file_d var100 : 50 < 200 -# $VAR1 = { -# 'test_dir1 file_b var17' => { -# 'RMS' => '0.314', -# 'Variable' => 'var17', -# 'Filename' => 'file_b', -# 'FILLDIFF' => '', -# 'Dir' => 'test_dir1', -# 'RMS_NORM' => '3.14' -# }, -# 'test_dir2 file_d var100' => { -# 'Dir' => 'test_dir2', -# 'RMS_NORM' => 200, -# 'Filename' => 'file_d', -# 'Variable' => 'var100', -# 'RMS' => '9' -# }, -# 'test_dir1 file_b var1' => { -# 'Filename' => 'file_b', -# 'RMS_NORM' => '42', -# 'FILLDIFF' => '', -# 'Dir' => 'test_dir1', -# 'RMS' => '4200', -# 'Variable' => 'var1' -# }, -# 'test_dir2 file_c var43' => { -# 'Variable' => 'var43', -# 'DIMSIZEDIFF' => '', -# 'Dir' => 'test_dir2', -# 'Filename' => 'file_c' -# }, -# 'test_dir2 file_c var42' => { -# 'Filename' => 'file_c', -# 'Dir' => 'test_dir2', -# 'FILLDIFF' => '', -# 'Variable' => 'var42' -# } -# }; - - -#----------------------------------------------------------------------- -# Testing the print routines -#----------------------------------------------------------------------- - -# Add the following to the above test code: - -# my $widths_hash = { Dir => 40, Filename => 40, Variable => 40 }; -# print_results_by_test("testout.by_test", \%diffs, $widths_hash); -# print_results_by_rms("testout.by_rms", \%diffs, $widths_hash); - -# This should give: - -# $ cat testout.by_rms -# test_dir2 : file_d : var100 : RMS 9 : RMS_NORM 200 -# test_dir1 : file_b : var1 : FILLDIFF : RMS 4200 : RMS_NORM 42 -# test_dir1 : file_b : var17 : FILLDIFF : RMS 0.314 : RMS_NORM 3.14 -# test_dir2 : file_c : var42 : FILLDIFF -# test_dir2 : file_c : var43 : DIMSIZEDIFF -# $ cat testout.by_test -# test_dir1 : file_b : var1 : FILLDIFF : RMS 4200 : RMS_NORM 42 -# test_dir1 : file_b : var17 : FILLDIFF : RMS 0.314 : RMS_NORM 3.14 -# ================================================================================================================================================================================= -# test_dir2 : file_c : var42 : FILLDIFF -# test_dir2 : file_c : var43 : DIMSIZEDIFF -# ================================================================================================================================================================================= -# test_dir2 : file_d : var100 : RMS 9 : RMS_NORM 200 - - - -#======================================================================= -# Notes about testing: integration tests -#======================================================================= - -# Test the following - -# Note: can do these tests by running the cprnc tests and organizing -# outputs into particular directories. -# -# For each of these tests, sort the different output files and compare -# the sorted files to make sure the same info is in all output files; -# then look at one of the output files. -# -# - no RMS or FILLDIFFs at all (testid that just contains output from -# comparing control & copy) -# -# - some RMS and some FILLDIFFs, split across 2 directories, each with -# 2 cprnc files (this can be done by comparing the control file with -# diffs_in_fill.nc, diffs_in_vals.nc, diffs_in_vals_and_diffs_in_fill.nc -# and diffs_in_vals_and_fill.nc) -# -# - multiple RMS errors to test RMS sorting, split across 2 directories -# (this can be done by comparing the control file with four of the -# vals_differ_by_* files) diff --git a/CIME/non_py/cprnc/test_inputs/README b/CIME/non_py/cprnc/test_inputs/README deleted file mode 100644 index f1bdfbd94e6..00000000000 --- a/CIME/non_py/cprnc/test_inputs/README +++ /dev/null @@ -1,188 +0,0 @@ -This directory contains simple test inputs to test cprnc. - -All comparisons can be run by running the run_tests script in the -parent directory. Suggestion: run this once from the baseline -directory, then once from the new directory; compare against baselines -by doing a directory diff of the two directories, or with, e.g.: - - baseline_out=/PATH/TO/BASELINE/OUTPUT - new_out=/PATH/TO/NEW/OUTPUT - for fl in $baseline_out/*; do echo $fl; diff -a $fl $new_out/`basename $fl`; done > diff_report - -The files here are: - ---- FILES COMPARED AGAINST control.nc --- - -- copy.nc: copy of control file (i.e., no diffs) - -- diffs_in_vals.nc: one variable has differences in values - -- diffs_in_vals_and_extra_and_missing.nc: one variable has differences in - values; also, one variable is missing and there is an extra variable. Purpose - of this test is to make sure that this case is reported as a DIFFERENCE rather - than just a warning due to the missing fields. - -- diffs_in_fill.nc: one variable has differences in fill pattern - -- diffs_in_vals_and_diffs_in_fill.nc: one variable has differences in - values, another has differences in fill pattern - -- diffs_in_vals_and_fill.nc: a single variable has differences in both - values and fill pattern - -- extra_variables.nc: has two extra variables beyond those in control.nc - -- lon_differs.nc: number of longitude points differs - -- missing_variables.nc: missing two variables that are present in control.nc - -- vals_differ_by_1.1.nc: testvar has values equal to 1.1 times those - in the control file. This is useful for testing the relative - difference calculation. - - True values are the following (note that relative difference is - defined using a denominator of max(v1,v2)): - - - RMS diff: 0.6204837 (printed as 6.2e-1) - - avg rel diff: 0.0909091 (printed as 9.1e-2) - - avg decimal digits: 1.041393 (printed as 1.0) - - worst decimal digits: 1.041393 (printed as 1.0) - -- vals_differ_by_1.1_somewhere.nc: similar to vals_differ_by_1.1.nc, - but now only a single value differs by a factor of 1.1 - - True values are the following (note that relative difference is - defined using a denominator of max(v1,v2)): - - - RMS diff: 0.3162278 (printed as 3.2e-1) - - avg rel diff: 0.009090909 (printed as 9.1e-3) - - avg decimal digits: 1.041393 (printed as 1.0) [note that the - average here ignores the indices with no difference] - - worst decimal digits: 1.041393 (printed as 1.0) - -- vals_differ_by_varying_amounts.nc: testvar has values equal to 1, - 1.01, 1.02, ..., 1.09 times those in the control file. This is - useful for testing the relative difference calculation using more - complex differences. - - True values are the following (note that relative difference is - defined using a denominator of max(v1,v2)): - - - RMS diff: 0.4434862 (printed as 4.4e-1) - - avg rel diff: 0.04233828 (printed as 4.2e-2) - - avg decimal digits: 1.403306 (printed as 1.4) [note that the - average here normalizes by 9 rather than 10, since the first index - has a relative difference of 0] - - worst decimal digits: 1.083184 (printed as 1.1) - -- vals_differ_by_varying_amounts2.nc: First 8 values of testvar are - identical to control; 9th is control * (1-1e-3), 10th is control * - (1-1e-5). This is the same as the example given in ../README. - - True values are the following: - - - RMS diff: 0.002846226 (printed as 2.8e-3) - - avg rel diff: 0.000101 (printed as 1.0e-4) - - avg decimal digits: 4.0 - - worst decimal digits: 3.0 - ---- FILES COMPARED AGAINST control_int.nc --- - -Note: This file is the same as control.nc, but stores the main variables -as integers rather than reals. - -- int_missing.nc: A variable is missing. The point of this test is to - cover code that resulted in https://github.com/ESMCI/cime/issues/3661. - ---- FILES COMPARED AGAINST control_multipleTimes_someTimeless.nc --- - -Note: This file has some variables with a time dimension, some -without. The time dimension has multiple times, in order to make sure -that the variables with vs. without the time dimension really are -treated differently. Also, a variable without time appears first, in -order to make sure that cprnc doesn't rely on there being a variable -with time first. - -- multipleTimes_someTimeless_diffs_in_vals_and_fill.nc: one variable - with a time dimension has differences in both values and fill - pattern (in time 2); and one variable without a time dimension has - differences in both values and fill pattern. The differences are the - same for both variables (e.g., RMS errors should be the same for - both). - -- multipleTimes_someTimeless_extra_and_missing.nc: two timeless - variables are missing and there is one extra timeless - variable. Purpose of this test is to make sure that the results are - reported as IDENTICAL when the only diffs in field lists are variables - without an unlimited dimension (in a file that has an unlimited - dimension). - ---- FILES COMPARED AGAINST control_noTime.nc --- - -Note: This file has no time (unlimited) dimension. - -- noTime_diffs_in_vals_and_fill.nc: a single variable has differences - in both values and fill pattern - -- noTime_extra_and_missing.nc: two variables are missing and there is - one extra variable. Purpose of this test is to make sure that even - missing fields without an unlimited dimension trigger a DIFFER result - if the file doesn't have an unlimited dimension to begin with. - ---- FILES COMPARED AGAINST control_0d.nc --- - -Note: This file has two 0-d variables - -- diffs_0d.nc: diffs in both 0-d variables (int & real) - - ---- FILES COMPARED AGAINST cpl.hi.subset.control.nc --- - -Note: This file is a subset of a standard cpl hist file (as of May, 2013). - -- cpl.hi.subset.test.nc: some variables are the same, some differ - - ---- FILES COMPARED AGAINST clm2.h0.subset.control.nc --- - -Note: This file is a subset of a standard clm hist file (as of May, 2013). - -- clm2.h0.subset.test.nc: some variables are the same, some differ - - ---- FILES COMPARED AGAINST clm2.h1.subset.control.nc --- - -Note: This file is a subset of a standard clm 1-d hist file -(dov2xy=false) (as of May, 2013). Note that it also includes two -times. - -- clm2.h1.subset.test.nc: some variables are the same, some - differ. Note that this includes identical & different integer - variables, identical & different real-valued variables, and - variables with different spatial dimensions (e.g., landunit, pft, - and lat x lon). - ---- FILES COMPARED AGAINST control_attributes.nc --- - -Note: This file is like control.nc, but contains some global attributes - -- diffs_in_attribute.nc: one global attribute is the same, one differs; in - addition, one global attribute is missing, and there is a new one on this file - that is not present on the control.nc file - ---- FILES COMPARED AGAINST control_char.nc --- - -Note: This file just has a character variable, in order to test what is output -for character variables (which cannot be analyzed). - -- copy_char.nc: identical to control_char.nc - ---- FILES COMPARED AGAINST control_floatDoubleNan.nc --- - -Note: This file has a float variable, a double variable, and a double -variable with a NaN value. Its initial purpose is for testing -comparisons involving NaNs. - -- diffs_in_nans.nc: a float and double variable each have a NaN where - the control file does not, and another double variable has a NaN - only where the control file also has a NaN diff --git a/CIME/non_py/cprnc/test_inputs/clm2.h0.subset.control.nc b/CIME/non_py/cprnc/test_inputs/clm2.h0.subset.control.nc deleted file mode 100644 index 1ffd6474f60..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/clm2.h0.subset.control.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/clm2.h0.subset.test.nc b/CIME/non_py/cprnc/test_inputs/clm2.h0.subset.test.nc deleted file mode 100644 index 41a08ce7bf7..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/clm2.h0.subset.test.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/clm2.h1.subset.control.nc b/CIME/non_py/cprnc/test_inputs/clm2.h1.subset.control.nc deleted file mode 100644 index 2d96f1fdc74..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/clm2.h1.subset.control.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/clm2.h1.subset.test.nc b/CIME/non_py/cprnc/test_inputs/clm2.h1.subset.test.nc deleted file mode 100644 index db606c4b135..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/clm2.h1.subset.test.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control.nc b/CIME/non_py/cprnc/test_inputs/control.nc deleted file mode 100644 index d9c0ce6a5c4..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_0d.nc b/CIME/non_py/cprnc/test_inputs/control_0d.nc deleted file mode 100644 index d48586f2282..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_0d.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_attributes.nc b/CIME/non_py/cprnc/test_inputs/control_attributes.nc deleted file mode 100644 index a26bc946415..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_attributes.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_char.nc b/CIME/non_py/cprnc/test_inputs/control_char.nc deleted file mode 100644 index 7b4567908ea..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_char.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_floatDoubleNan.nc b/CIME/non_py/cprnc/test_inputs/control_floatDoubleNan.nc deleted file mode 100644 index 1e99d3281e6..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_floatDoubleNan.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_int.nc b/CIME/non_py/cprnc/test_inputs/control_int.nc deleted file mode 100644 index 13913cf90e3..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_int.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_multipleTimes_someTimeless.nc b/CIME/non_py/cprnc/test_inputs/control_multipleTimes_someTimeless.nc deleted file mode 100644 index 9ebf4579c92..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_multipleTimes_someTimeless.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/control_noTime.nc b/CIME/non_py/cprnc/test_inputs/control_noTime.nc deleted file mode 100644 index 92be43053d0..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/control_noTime.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/copy.nc b/CIME/non_py/cprnc/test_inputs/copy.nc deleted file mode 100644 index d9c0ce6a5c4..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/copy.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/copy_char.nc b/CIME/non_py/cprnc/test_inputs/copy_char.nc deleted file mode 100644 index 7b4567908ea..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/copy_char.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/cpl.hi.subset.control.nc b/CIME/non_py/cprnc/test_inputs/cpl.hi.subset.control.nc deleted file mode 100644 index c83d8cc499d..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/cpl.hi.subset.control.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/cpl.hi.subset.test.nc b/CIME/non_py/cprnc/test_inputs/cpl.hi.subset.test.nc deleted file mode 100644 index 3fdd331ba42..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/cpl.hi.subset.test.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_0d.nc b/CIME/non_py/cprnc/test_inputs/diffs_0d.nc deleted file mode 100644 index db275131129..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_0d.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_attribute.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_attribute.nc deleted file mode 100644 index b81f05122af..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_attribute.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_fill.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_fill.nc deleted file mode 100644 index 07b62e33948..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_fill.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_nans.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_nans.nc deleted file mode 100644 index 732bf56896f..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_nans.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_vals.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_vals.nc deleted file mode 100644 index 16411c4881a..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_vals.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_diffs_in_fill.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_diffs_in_fill.nc deleted file mode 100644 index 9b7adc86763..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_diffs_in_fill.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_extra_and_missing.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_extra_and_missing.nc deleted file mode 100644 index e29e072a24c..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_extra_and_missing.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_fill.nc b/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_fill.nc deleted file mode 100644 index 6db4782e6ae..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/diffs_in_vals_and_fill.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/extra_variables.nc b/CIME/non_py/cprnc/test_inputs/extra_variables.nc deleted file mode 100644 index 2d33ab90796..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/extra_variables.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/int_missing.nc b/CIME/non_py/cprnc/test_inputs/int_missing.nc deleted file mode 100644 index 139b358d57f..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/int_missing.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/lon_differs.nc b/CIME/non_py/cprnc/test_inputs/lon_differs.nc deleted file mode 100644 index bb429246912..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/lon_differs.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/missing_variables.nc b/CIME/non_py/cprnc/test_inputs/missing_variables.nc deleted file mode 100644 index 71d9ae2d082..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/missing_variables.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/multipleTimes_someTimeless_diffs_in_vals_and_fill.nc b/CIME/non_py/cprnc/test_inputs/multipleTimes_someTimeless_diffs_in_vals_and_fill.nc deleted file mode 100644 index 7edf3a495d3..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/multipleTimes_someTimeless_diffs_in_vals_and_fill.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/multipleTimes_someTimeless_extra_and_missing.nc b/CIME/non_py/cprnc/test_inputs/multipleTimes_someTimeless_extra_and_missing.nc deleted file mode 100644 index d2718a86de5..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/multipleTimes_someTimeless_extra_and_missing.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/noTime_diffs_in_vals_and_fill.nc b/CIME/non_py/cprnc/test_inputs/noTime_diffs_in_vals_and_fill.nc deleted file mode 100644 index c3006345ed7..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/noTime_diffs_in_vals_and_fill.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/noTime_extra_and_missing.nc b/CIME/non_py/cprnc/test_inputs/noTime_extra_and_missing.nc deleted file mode 100644 index c6c2d790900..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/noTime_extra_and_missing.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/vals_differ_by_1.1.nc b/CIME/non_py/cprnc/test_inputs/vals_differ_by_1.1.nc deleted file mode 100644 index ddc52582461..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/vals_differ_by_1.1.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/vals_differ_by_1.1_somewhere.nc b/CIME/non_py/cprnc/test_inputs/vals_differ_by_1.1_somewhere.nc deleted file mode 100644 index 703507681b5..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/vals_differ_by_1.1_somewhere.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/vals_differ_by_varying_amounts.nc b/CIME/non_py/cprnc/test_inputs/vals_differ_by_varying_amounts.nc deleted file mode 100644 index f42d2952af2..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/vals_differ_by_varying_amounts.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/test_inputs/vals_differ_by_varying_amounts2.nc b/CIME/non_py/cprnc/test_inputs/vals_differ_by_varying_amounts2.nc deleted file mode 100644 index 4167b2d121b..00000000000 Binary files a/CIME/non_py/cprnc/test_inputs/vals_differ_by_varying_amounts2.nc and /dev/null differ diff --git a/CIME/non_py/cprnc/utils.F90 b/CIME/non_py/cprnc/utils.F90 deleted file mode 100644 index 9bf0d5d82fc..00000000000 --- a/CIME/non_py/cprnc/utils.F90 +++ /dev/null @@ -1,85 +0,0 @@ -module utils - use filestruct, only : dim_t -contains - -subroutine get_dimname_str(ndims,dimids,dims,dimname_str) - integer, intent(in) :: ndims - integer, intent(in) :: dimids(:) - type(dim_t) :: dims(:) - character(len=*),intent(out) :: dimname_str - - integer :: dlen - integer :: j - - dimname_str = ' ' - - if(ndims>0) then - dimname_str(1:1) = '(' - dlen=2 - - do j=1,ndims - dimname_str(dlen:) = trim(dims(dimids(j))%name)//',' - dlen=dlen+ len_trim(dims(dimids(j))%name) + 1 - end do - dimname_str(dlen-1:dlen-1) = ')' - end if - - -end subroutine get_dimname_str - -subroutine get_dim_str(ndims,loc,dim_str) - integer, intent(in) :: ndims - integer, intent(in) :: loc(:) - character(len=*),intent(out) :: dim_str - - integer :: dlen - integer :: j - - dim_str = ' ' - - if(ndims>0) then - dim_str(1:1) = '(' - dlen=2 - - do j=1,ndims - write(dim_str(dlen:),'(i6,a)') loc(j),',' - - dlen=len_trim(dim_str)+1 - end do - dim_str(dlen-1:dlen-1) = ')' - end if - - -end subroutine get_dim_str - - - -subroutine checknf90(ierr,returnflag,err_str) - use netcdf, only : nf90_noerr, nf90_strerror - integer, intent(in) :: ierr - logical, optional, intent(in) :: returnflag - character(len=*), optional, intent(in) :: err_str - - if(ierr/=NF90_NOERR) then - print *, trim(nf90_strerror(ierr)) - if(present(err_str)) then - print *, trim(err_str) - end if - if(present(returnflag)) then - if(returnflag) return - end if -#ifdef AIX - call xl__trbk() -#endif - stop - - end if - - - -end subroutine checknf90 - - - - -end module utils diff --git a/CIME/non_py/src/timing/Makefile b/CIME/non_py/src/timing/Makefile index 89f3b4a7bed..7b0505fbffa 100644 --- a/CIME/non_py/src/timing/Makefile +++ b/CIME/non_py/src/timing/Makefile @@ -52,7 +52,11 @@ ifeq ($(strip $(MPILIB)), mpi-serial) FC := $(SFC) MPIFC := $(SFC) MPICC := $(SCC) - INCLDIR += -I$(GPTL_LIBDIR)/../mct/mpi-serial + ifdef MPI_SERIAL_PATH + INCLDIR += -I$(MPI_SERIAL_PATH)/include + else + INCLDIR += -I$(GPTL_LIBDIR)/../mct/mpi-serial + endif else CC := $(MPICC) FC := $(MPIFC) diff --git a/CIME/tests/scripts_regression_tests.py b/CIME/tests/scripts_regression_tests.py index 039f13c7718..66f4c015298 100755 --- a/CIME/tests/scripts_regression_tests.py +++ b/CIME/tests/scripts_regression_tests.py @@ -44,6 +44,7 @@ from CIME.provenance import get_test_success, save_test_success from CIME import utils from CIME.tests.base import BaseTestCase +from CIME.config import Config os.environ["CIME_GLOBAL_WALLTIME"] = "0:05:00" @@ -151,6 +152,9 @@ def configure_tests( ): config = CIME.utils.get_cime_config() + customize_path = os.path.join(utils.get_src_root(), "cime_config", "customize") + Config.load(customize_path) + if timeout: BaseTestCase.GLOBAL_TIMEOUT = str(timeout) diff --git a/CIME/tests/test_sys_cime_case.py b/CIME/tests/test_sys_cime_case.py index 6ca9a92168e..dabea7d9600 100644 --- a/CIME/tests/test_sys_cime_case.py +++ b/CIME/tests/test_sys_cime_case.py @@ -236,7 +236,7 @@ def test_cime_case_build_threaded_1(self): ) with Case(casedir, read_only=False) as case: - build_threaded = case.get_value("SMP_PRESENT") + build_threaded = case.get_value("BUILD_THREADED") self.assertFalse(build_threaded) build_threaded = case.get_build_threaded() @@ -254,7 +254,7 @@ def test_cime_case_build_threaded_2(self): ) with Case(casedir, read_only=False) as case: - build_threaded = case.get_value("SMP_PRESENT") + build_threaded = case.get_value("BUILD_THREADED") self.assertTrue(build_threaded) build_threaded = case.get_build_threaded() diff --git a/CIME/tests/test_sys_create_newcase.py b/CIME/tests/test_sys_create_newcase.py index 1d240c9ee1f..a0f07a001b6 100644 --- a/CIME/tests/test_sys_create_newcase.py +++ b/CIME/tests/test_sys_create_newcase.py @@ -109,10 +109,12 @@ def test_a_createnewcase(self): new_batch_command = case.get_value( "BATCH_COMMAND_FLAGS", subgroup="case.run" ) - self.assertTrue( - "fred" in new_batch_command, - msg="Failed to update JOB_QUEUE in BATCH_COMMAND_FLAGS", - ) + self.assertTrue( + "fred" in new_batch_command, + msg="Failed to update JOB_QUEUE in BATCH_COMMAND_FLAGS {}".format( + new_batch_command + ), + ) # Trying to set values outside of context manager should fail case = Case(testdir, read_only=False) diff --git a/CIME/tests/test_sys_test_scheduler.py b/CIME/tests/test_sys_test_scheduler.py index b53809ba306..3dfd4b62124 100755 --- a/CIME/tests/test_sys_test_scheduler.py +++ b/CIME/tests/test_sys_test_scheduler.py @@ -20,22 +20,22 @@ def test_chksum(self, strftime): # pylint: disable=unused-argument self.skipTest("Skipping chksum test. Depends on CESM settings") ts = test_scheduler.TestScheduler( - ["SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu"], - machine_name="cori-haswell", + ["SEQ_Ln9.f19_g16_rx1.A.perlmutter_gnu"], + machine_name="perlmutter", chksum=True, test_root="/tests", ) with mock.patch.object(ts, "_shell_cmd_for_phase") as _shell_cmd_for_phase: ts._run_phase( - "SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu" + "SEQ_Ln9.f19_g16_rx1.A.perlmutter_gnu" ) # pylint: disable=protected-access _shell_cmd_for_phase.assert_called_with( - "SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu", + "SEQ_Ln9.f19_g16_rx1.A.perlmutter_gnu", "./case.submit --skip-preview-namelist --chksum", "RUN", - from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.cori-haswell_gnu.00:00:00", + from_dir="/tests/SEQ_Ln9.f19_g16_rx1.A.perlmutter_gnu.00:00:00", ) def test_a_phases(self): diff --git a/CIME/tests/test_sys_unittest.py b/CIME/tests/test_sys_unittest.py index af43654454c..238c7355469 100755 --- a/CIME/tests/test_sys_unittest.py +++ b/CIME/tests/test_sys_unittest.py @@ -16,35 +16,42 @@ def setUpClass(cls): cls._testroot = os.path.join(cls.TEST_ROOT, "TestUnitTests") cls._testdirs = [] - def _has_unit_test_support(self): - if self.TEST_COMPILER is None: - compiler = self.MACHINE.get_default_compiler() - else: - compiler = self.TEST_COMPILER + def setUp(self): + super().setUp() + + self._driver = utils.get_cime_default_driver() + self._has_pfunit = self._has_unit_test_support() - mach = self.MACHINE.get_machine_name() + def _has_unit_test_support(self): cmake_macros_dir = Files().get_value("CMAKE_MACROS_DIR") macros_to_check = [ - os.path.join(cmake_macros_dir, "{}_{}.cmake".format(compiler, mach)), - os.path.join(cmake_macros_dir, "{}.cmake".format(mach)), os.path.join( - os.environ.get("HOME"), ".cime", "{}_{}.cmake".format(compiler, mach) + cmake_macros_dir, + "{}_{}.cmake".format(self._compiler, self._machine), + ), + os.path.join(cmake_macros_dir, "{}.cmake".format(self._machine)), + os.path.join( + os.environ.get("HOME"), + ".cime", + "{}_{}.cmake".format(self._compiler, self._machine), + ), + os.path.join( + os.environ.get("HOME"), ".cime", "{}.cmake".format(self._machine) ), - os.path.join(os.environ.get("HOME"), ".cime", "{}.cmake".format(mach)), ] for macro_to_check in macros_to_check: if os.path.exists(macro_to_check): macro_text = open(macro_to_check, "r").read() - - return "PFUNIT_PATH" in macro_text + if "PFUNIT_PATH" in macro_text: + return True return False def test_a_unit_test(self): cls = self.__class__ - if not self._has_unit_test_support(): + if not self._has_pfunit: self.skipTest( "Skipping TestUnitTest - PFUNIT_PATH not found for the default compiler on this machine" ) @@ -59,8 +66,7 @@ def test_a_unit_test(self): test_spec_dir = os.path.join( os.path.dirname(unit_test_tool), "Examples", "interpolate_1d", "tests" ) - args = "--build-dir {} --test-spec-dir {}".format(test_dir, test_spec_dir) - args += " --machine {}".format(self.MACHINE.get_machine_name()) + args = f"--build-dir {test_dir} --test-spec-dir {test_spec_dir} --machine {self._machine} --compiler {self._compiler} --comp-interface {self._driver}" utils.run_cmd_no_fail("{} {}".format(unit_test_tool, args)) cls._do_teardown.append(test_dir) @@ -83,8 +89,7 @@ def test_b_cime_f90_unit_tests(self): test_spec_dir, "scripts", "fortran_unit_testing", "run_tests.py" ) ) - args = "--build-dir {} --test-spec-dir {}".format(test_dir, test_spec_dir) - args += " --machine {}".format(self.MACHINE.get_machine_name()) + args = f"--build-dir {test_dir} --test-spec-dir {test_spec_dir} --machine {self._machine} --compiler {self._compiler} --comp-interface {self._driver}" utils.run_cmd_no_fail("{} {}".format(unit_test_tool, args)) cls._do_teardown.append(test_dir) diff --git a/CIME/tests/test_unit_baselines_performance.py b/CIME/tests/test_unit_baselines_performance.py index 1422f3412b8..1564541ba9a 100644 --- a/CIME/tests/test_unit_baselines_performance.py +++ b/CIME/tests/test_unit_baselines_performance.py @@ -28,22 +28,9 @@ def create_mock_case(tempdir, get_latest_cpl_logs=None): class TestUnitBaselinesPerformance(unittest.TestCase): - @mock.patch("CIME.baselines.performance._perf_get_memory") - def test_perf_get_memory_default_no_value(self, _perf_get_memory): - _perf_get_memory.return_value = None - - case = mock.MagicMock() - - config = mock.MagicMock() - - config.perf_get_memory.side_effect = AttributeError - - with self.assertRaises(RuntimeError): - performance.perf_get_memory(case, config) - @mock.patch("CIME.baselines.performance._perf_get_memory") def test_perf_get_memory_default(self, _perf_get_memory): - _perf_get_memory.return_value = [(1, 1000)] + _perf_get_memory.return_value = ("1000", "a") case = mock.MagicMock() @@ -53,35 +40,22 @@ def test_perf_get_memory_default(self, _perf_get_memory): mem = performance.perf_get_memory(case, config) - assert mem == "1000" + assert mem == ("1000", "a") def test_perf_get_memory(self): case = mock.MagicMock() config = mock.MagicMock() - config.perf_get_memory.return_value = "1000" + config.perf_get_memory.return_value = ("1000", "a") mem = performance.perf_get_memory(case, config) - assert mem == "1000" - - @mock.patch("CIME.baselines.performance._perf_get_throughput") - def test_perf_get_throughput_default_no_value(self, _perf_get_throughput): - _perf_get_throughput.return_value = None - - case = mock.MagicMock() - - config = mock.MagicMock() - - config.perf_get_throughput.side_effect = AttributeError - - with self.assertRaises(RuntimeError): - performance.perf_get_throughput(case, config) + assert mem == ("1000", "a") @mock.patch("CIME.baselines.performance._perf_get_throughput") def test_perf_get_throughput_default(self, _perf_get_throughput): - _perf_get_throughput.return_value = 100 + _perf_get_throughput.return_value = ("100", "a") case = mock.MagicMock() @@ -91,18 +65,18 @@ def test_perf_get_throughput_default(self, _perf_get_throughput): tput = performance.perf_get_throughput(case, config) - assert tput == "100" + assert tput == ("100", "a") def test_perf_get_throughput(self): case = mock.MagicMock() config = mock.MagicMock() - config.perf_get_throughput.return_value = "100" + config.perf_get_throughput.return_value = ("100", "a") tput = performance.perf_get_throughput(case, config) - assert tput == "100" + assert tput == ("100", "a") def test_get_cpl_throughput_no_file(self): throughput = performance.get_cpl_throughput("/tmp/cpl.log") @@ -155,35 +129,29 @@ def test_get_cpl_mem_usage(self, isfile): def test_read_baseline_file_multi_line(self): with mock.patch( "builtins.open", - mock.mock_open(read_data="#comment about data\n1000.0\n2000.0\n"), + mock.mock_open( + read_data="sha:1df0 date:2023 1000.0\nsha:3b05 date:2023 2000.0" + ), ) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == "1000.0\n2000.0" + assert baseline == "sha:1df0 date:2023 1000.0\nsha:3b05 date:2023 2000.0" def test_read_baseline_file_content(self): with mock.patch( - "builtins.open", mock.mock_open(read_data="1000.0") + "builtins.open", mock.mock_open(read_data="sha:1df0 date:2023 1000.0") ) as mock_file: baseline = performance.read_baseline_file("/tmp/cpl-mem.log") mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == "1000.0" - - def test_read_baseline_file(self): - with mock.patch("builtins.open", mock.mock_open(read_data="")) as mock_file: - baseline = performance.read_baseline_file("/tmp/cpl-mem.log") - - mock_file.assert_called_with("/tmp/cpl-mem.log") - assert baseline == "" + assert baseline == "sha:1df0 date:2023 1000.0" def test_write_baseline_file(self): with mock.patch("builtins.open", mock.mock_open()) as mock_file: performance.write_baseline_file("/tmp/cpl-tput.log", "1000") - mock_file.assert_called_with("/tmp/cpl-tput.log", "w") - mock_file.return_value.write.assert_called_with("1000") + mock_file.assert_called_with("/tmp/cpl-tput.log", "a") @mock.patch("CIME.baselines.performance.get_cpl_throughput") @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") @@ -193,9 +161,8 @@ def test__perf_get_throughput(self, get_latest_cpl_logs, get_cpl_throughput): with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - tput = performance._perf_get_throughput(case) - - assert tput == None + with self.assertRaises(RuntimeError): + performance._perf_get_throughput(case) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") @@ -205,9 +172,8 @@ def test__perf_get_memory_override(self, get_latest_cpl_logs, get_cpl_mem_usage) with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = performance._perf_get_memory(case, "/tmp/override") - - assert mem == None + with self.assertRaises(RuntimeError): + performance._perf_get_memory(case, "/tmp/override") @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @mock.patch("CIME.baselines.performance.get_latest_cpl_logs") @@ -217,9 +183,8 @@ def test__perf_get_memory(self, get_latest_cpl_logs, get_cpl_mem_usage): with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) - mem = performance._perf_get_memory(case) - - assert mem == None + with self.assertRaises(RuntimeError): + performance._perf_get_memory(case) @mock.patch("CIME.baselines.performance.write_baseline_file") @mock.patch("CIME.baselines.performance.perf_get_memory") @@ -270,9 +235,9 @@ def test_write_baseline_runtimeerror( def test_perf_write_baseline( self, perf_get_throughput, perf_get_memory, write_baseline_file ): - perf_get_throughput.return_value = "100" + perf_get_throughput.return_value = ("100", "a") - perf_get_memory.return_value = "1000" + perf_get_memory.return_value = ("1000", "a") with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir) @@ -281,8 +246,12 @@ def test_perf_write_baseline( perf_get_throughput.assert_called() perf_get_memory.assert_called() - write_baseline_file.assert_any_call(str(baseline_root / "cpl-tput.log"), "100") - write_baseline_file.assert_any_call(str(baseline_root / "cpl-mem.log"), "1000") + write_baseline_file.assert_any_call( + str(baseline_root / "cpl-tput.log"), "100", "a" + ) + write_baseline_file.assert_any_call( + str(baseline_root / "cpl-mem.log"), "1000", "a" + ) @mock.patch("CIME.baselines.performance._perf_get_throughput") @mock.patch("CIME.baselines.performance.read_baseline_file") @@ -315,7 +284,7 @@ def test_perf_compare_throughput_baseline_no_baseline( ): read_baseline_file.return_value = "" - _perf_get_throughput.return_value = 504 + _perf_get_throughput.return_value = ("504", "a") with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -336,7 +305,7 @@ def test_perf_compare_throughput_baseline_no_baseline( assert below_tolerance is None assert ( comment - == "Could not compare throughput to baseline, as basline had no value." + == "Could not compare throughput to baseline, as baseline had no value." ) @mock.patch("CIME.baselines.performance._perf_get_throughput") @@ -347,7 +316,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( ): read_baseline_file.return_value = "500" - _perf_get_throughput.return_value = 504 + _perf_get_throughput.return_value = ("504", "a") with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -368,7 +337,7 @@ def test_perf_compare_throughput_baseline_no_tolerance( assert below_tolerance assert ( comment - == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + == "TPUTCOMP: Throughput changed by -0.80%: baseline=500.000 sypd, tolerance=10%, current=504.000 sypd" ) @mock.patch("CIME.baselines.performance._perf_get_throughput") @@ -379,7 +348,7 @@ def test_perf_compare_throughput_baseline_above_threshold( ): read_baseline_file.return_value = "1000" - _perf_get_throughput.return_value = 504 + _perf_get_throughput.return_value = ("504", "a") with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -399,7 +368,8 @@ def test_perf_compare_throughput_baseline_above_threshold( assert not below_tolerance assert ( - comment == "Error: TPUTCOMP: Computation time increase > 5% from baseline" + comment + == "Error: TPUTCOMP: Throughput changed by 49.60%: baseline=1000.000 sypd, tolerance=5%, current=504.000 sypd" ) @mock.patch("CIME.baselines.performance._perf_get_throughput") @@ -410,7 +380,7 @@ def test_perf_compare_throughput_baseline( ): read_baseline_file.return_value = "500" - _perf_get_throughput.return_value = 504 + _perf_get_throughput.return_value = ("504", "a") with tempfile.TemporaryDirectory() as tempdir: case, _, _, baseline_root = create_mock_case(tempdir, get_latest_cpl_logs) @@ -431,7 +401,7 @@ def test_perf_compare_throughput_baseline( assert below_tolerance assert ( comment - == "TPUTCOMP: Computation time changed by -0.80% relative to baseline" + == "TPUTCOMP: Throughput changed by -0.80%: baseline=500.000 sypd, tolerance=5%, current=504.000 sypd" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -466,7 +436,7 @@ def test_perf_compare_memory_baseline_no_baseline( assert below_tolerance assert ( comment - == "MEMCOMP: Memory usage highwater has changed by 0.00% relative to baseline" + == "MEMCOMP: Memory usage highwater changed by 0.00%: baseline=0.000 MB, tolerance=5%, current=1003.000 MB" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -557,7 +527,7 @@ def test_perf_compare_memory_baseline_no_tolerance( assert below_tolerance assert ( comment - == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + == "MEMCOMP: Memory usage highwater changed by 0.30%: baseline=1000.000 MB, tolerance=10%, current=1003.000 MB" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -592,7 +562,7 @@ def test_perf_compare_memory_baseline_above_threshold( assert not below_tolerance assert ( comment - == "Error: Memory usage increase >5% from baseline's 1000.000000 to 2003.000000" + == "Error: MEMCOMP: Memory usage highwater changed by 100.30%: baseline=1000.000 MB, tolerance=5%, current=2003.000 MB" ) @mock.patch("CIME.baselines.performance.get_cpl_mem_usage") @@ -627,7 +597,7 @@ def test_perf_compare_memory_baseline( assert below_tolerance assert ( comment - == "MEMCOMP: Memory usage highwater has changed by 0.30% relative to baseline" + == "MEMCOMP: Memory usage highwater changed by 0.30%: baseline=1000.000 MB, tolerance=5%, current=1003.000 MB" ) def test_get_latest_cpl_logs_found_multiple(self): diff --git a/CIME/tests/test_unit_bless_test_results.py b/CIME/tests/test_unit_bless_test_results.py index fe9003d1bd2..91315b30f8d 100644 --- a/CIME/tests/test_unit_bless_test_results.py +++ b/CIME/tests/test_unit_bless_test_results.py @@ -429,7 +429,7 @@ def test_bless_perf( ts = TestStatus.return_value ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" ts.get_overall_test_status.return_value = ("PASS", "RUN") - ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL", "FAIL"] + ts.get_status.side_effect = ["PASS", "PASS", "FAIL", "FAIL", "FAIL"] case = Case.return_value.__enter__.return_value @@ -470,7 +470,7 @@ def test_bless_memory_only( ts = TestStatus.return_value ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" ts.get_overall_test_status.return_value = ("PASS", "RUN") - ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL"] + ts.get_status.side_effect = ["PASS", "PASS", "FAIL", "FAIL"] case = Case.return_value.__enter__.return_value @@ -509,7 +509,7 @@ def test_bless_throughput_only( ts = TestStatus.return_value ts.get_name.return_value = "SMS.f19_g16.S.docker_gnu" ts.get_overall_test_status.return_value = ("PASS", "RUN") - ts.get_status.side_effect = ["PASS", "PASS", "PASS", "FAIL"] + ts.get_status.side_effect = ["PASS", "PASS", "FAIL", "FAIL"] case = Case.return_value.__enter__.return_value @@ -798,13 +798,21 @@ def test_baseline_root_none(self, get_test_status_files, TestStatus, Case): assert not success + @mock.patch("CIME.utils.get_current_branch") @mock.patch("CIME.bless_test_results.bless_namelists") @mock.patch("CIME.bless_test_results.Case") @mock.patch("CIME.bless_test_results.TestStatus") @mock.patch("CIME.bless_test_results.get_test_status_files") def test_baseline_name_none( - self, get_test_status_files, TestStatus, Case, bless_namelists + self, + get_test_status_files, + TestStatus, + Case, + bless_namelists, + get_current_branch, ): + get_current_branch.return_value = "master" + bless_namelists.return_value = (True, "") get_test_status_files.return_value = [ diff --git a/CIME/tests/test_unit_case.py b/CIME/tests/test_unit_case.py index 8a1190456c2..b14458a8dea 100755 --- a/CIME/tests/test_unit_case.py +++ b/CIME/tests/test_unit_case.py @@ -23,19 +23,11 @@ class TestCaseSubmit(unittest.TestCase): def test_check_case(self): case = mock.MagicMock() # get_value arguments TEST, COMP_WAV, COMP_INTERFACE, BUILD_COMPLETE - case.get_value.side_effect = [False, "", "", True] + case.get_value.side_effect = ["", "", True] case_submit.check_case(case, chksum=True) case.check_all_input_data.assert_called_with(chksum=True) - def test_check_case_test(self): - case = mock.MagicMock() - # get_value arguments TEST, COMP_WAV, COMP_INTERFACE, BUILD_COMPLETE - case.get_value.side_effect = [True, "", "", True] - case_submit.check_case(case, chksum=True) - - case.check_all_input_data.assert_not_called() - @mock.patch("CIME.case.case_submit.lock_file") @mock.patch("CIME.case.case_submit.unlock_file") @mock.patch("os.path.basename") @@ -86,6 +78,7 @@ def test_submit( batch_args=None, workflow=True, chksum=True, + dryrun=False, ) @@ -232,14 +225,14 @@ def test_copy( self.srcroot, "A", "f19_g16_rx1", - machine_name="cori-haswell", + machine_name="perlmutter", ) # Check that they're all called configure.assert_called_with( "A", "f19_g16_rx1", - machine_name="cori-haswell", + machine_name="perlmutter", project=None, pecount=None, compiler=None, @@ -309,14 +302,14 @@ def test_create( self.srcroot, "A", "f19_g16_rx1", - machine_name="cori-haswell", + machine_name="perlmutter", ) # Check that they're all called configure.assert_called_with( "A", "f19_g16_rx1", - machine_name="cori-haswell", + machine_name="perlmutter", project=None, pecount=None, compiler=None, diff --git a/CIME/tests/test_unit_case_run.py b/CIME/tests/test_unit_case_run.py new file mode 100644 index 00000000000..8f188925d57 --- /dev/null +++ b/CIME/tests/test_unit_case_run.py @@ -0,0 +1,50 @@ +import unittest +from unittest import mock + +from CIME.utils import CIMEError +from CIME.case.case_run import TERMINATION_TEXT +from CIME.case.case_run import _post_run_check + + +def _case_post_run_check(): + case = mock.MagicMock() + + # RUNDIR, COMP_INTERFACE, COMP_CPL, COMP_ATM, COMP_OCN, MULTI_DRIVER + case.get_value.side_effect = ("/tmp/run", "mct", "cpl", "satm", "socn", False) + + # COMP_CLASSES + case.get_values.return_value = ("CPL", "ATM", "OCN") + + return case + + +class TestCaseSubmit(unittest.TestCase): + @mock.patch("os.stat") + @mock.patch("os.path.isfile") + def test_post_run_check(self, isfile, stat): + isfile.return_value = True + + stat.return_value.st_size = 1024 + + # no exceptions means success + for x in TERMINATION_TEXT: + case = _case_post_run_check() + + with mock.patch("builtins.open", mock.mock_open(read_data=x)) as mock_file: + _post_run_check(case, "1234") + + @mock.patch("os.stat") + @mock.patch("os.path.isfile") + def test_post_run_check_no_termination(self, isfile, stat): + isfile.return_value = True + + stat.return_value.st_size = 1024 + + case = _case_post_run_check() + + with self.assertRaises(CIMEError): + with mock.patch( + "builtins.open", + mock.mock_open(read_data="I DONT HAVE A TERMINATION MESSAGE"), + ) as mock_file: + _post_run_check(case, "1234") diff --git a/CIME/tests/test_unit_system_tests.py b/CIME/tests/test_unit_system_tests.py index 609460fe9c0..1c05bed45be 100644 --- a/CIME/tests/test_unit_system_tests.py +++ b/CIME/tests/test_unit_system_tests.py @@ -70,12 +70,12 @@ def create_mock_case(tempdir, idx=None, cpllog_data=None): class TestUnitSystemTests(unittest.TestCase): @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_get_memory_list") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_runtime_error( self, get_latest_cpl_logs, - _perf_get_memory, + perf_get_memory_list, append_testlog, load_coupler_customization, ): @@ -83,7 +83,7 @@ def test_check_for_memleak_runtime_error( AttributeError ) - _perf_get_memory.side_effect = RuntimeError + perf_get_memory_list.side_effect = RuntimeError with tempfile.TemporaryDirectory() as tempdir: caseroot = Path(tempdir) / "caseroot" @@ -120,12 +120,12 @@ def test_check_for_memleak_runtime_error( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_get_memory_list") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_not_enough_samples( self, get_latest_cpl_logs, - _perf_get_memory, + perf_get_memory_list, append_testlog, load_coupler_customization, ): @@ -133,7 +133,7 @@ def test_check_for_memleak_not_enough_samples( AttributeError ) - _perf_get_memory.return_value = [ + perf_get_memory_list.return_value = [ (1, 1000.0), (2, 0), ] @@ -173,12 +173,12 @@ def test_check_for_memleak_not_enough_samples( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_get_memory_list") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak_found( self, get_latest_cpl_logs, - _perf_get_memory, + perf_get_memory_list, append_testlog, load_coupler_customization, ): @@ -186,7 +186,7 @@ def test_check_for_memleak_found( AttributeError ) - _perf_get_memory.return_value = [ + perf_get_memory_list.return_value = [ (1, 1000.0), (2, 2000.0), (3, 3000.0), @@ -230,12 +230,12 @@ def test_check_for_memleak_found( @mock.patch("CIME.SystemTests.system_tests_common.load_coupler_customization") @mock.patch("CIME.SystemTests.system_tests_common.append_testlog") - @mock.patch("CIME.SystemTests.system_tests_common._perf_get_memory") + @mock.patch("CIME.SystemTests.system_tests_common.perf_get_memory_list") @mock.patch("CIME.SystemTests.system_tests_common.get_latest_cpl_logs") def test_check_for_memleak( self, get_latest_cpl_logs, - _perf_get_memory, + perf_get_memory_list, append_testlog, load_coupler_customization, ): @@ -243,7 +243,7 @@ def test_check_for_memleak( AttributeError ) - _perf_get_memory.return_value = [ + perf_get_memory_list.return_value = [ (1, 3040.0), (2, 3002.0), (3, 3030.0), @@ -509,16 +509,14 @@ def test_generate_baseline(self): with open(baseline_dir / "cpl-tput.log") as fd: lines = fd.readlines() - assert len(lines) == 2 - assert re.match("# sha:.* date:.*", lines[0]) - assert lines[1] == "719.635" + assert len(lines) == 1 + assert re.match("sha:.* date:.* (\d+\.\d+)", lines[0]) with open(baseline_dir / "cpl-mem.log") as fd: lines = fd.readlines() - assert len(lines) == 2 - assert re.match("# sha:.* date:.*", lines[0]) - assert lines[1] == "1673.89" + assert len(lines) == 1 + assert re.match("sha:.* date:.* (\d+\.\d+)", lines[0]) def test_kwargs(self): case = mock.MagicMock() diff --git a/CIME/tests/test_unit_xml_machines.py b/CIME/tests/test_unit_xml_machines.py index 831359380fe..d051a5d7d3e 100644 --- a/CIME/tests/test_unit_xml_machines.py +++ b/CIME/tests/test_unit_xml_machines.py @@ -78,10 +78,10 @@ /opt/ubuntu/pe/netcdf-hdf5parallel/4.8.1.3/gnu/9.1/ $SHELL{dirname $(dirname $(which pnetcdf_version))} - + 128M - + cores @@ -126,10 +126,10 @@ /opt/ubuntu/pe/netcdf-hdf5parallel/4.8.1.3/gnu/9.1/ $SHELL{dirname $(dirname $(which pnetcdf_version))} - + 128M - + cores diff --git a/CIME/utils.py b/CIME/utils.py index 1a32319c8d1..7471c2e4f4c 100644 --- a/CIME/utils.py +++ b/CIME/utils.py @@ -11,6 +11,7 @@ from argparse import Action from contextlib import contextmanager +# pylint: disable=deprecated-module from distutils import file_util # Return this error code if the scripts worked but tests failed @@ -201,7 +202,7 @@ def check_name(fullname, additional_chars=None, fullpath=False): False """ - chars = "+*?<>/{}[\]~`@:" # pylint: disable=anomalous-backslash-in-string + chars = r"+*?<>/{}[\]~`@:" if additional_chars is not None: chars += additional_chars if fullname.endswith("/"): @@ -1413,6 +1414,7 @@ def safe_copy(src_path, tgt_path, preserve_meta=True): tgt_path, preserve_mode=preserve_meta, preserve_times=preserve_meta, + verbose=0, ) else: # I am not the owner, just copy file contents @@ -1426,6 +1428,7 @@ def safe_copy(src_path, tgt_path, preserve_meta=True): tgt_path, preserve_mode=preserve_meta, preserve_times=preserve_meta, + verbose=0, ) # If src file was executable, then the tgt file should be too @@ -2734,3 +2737,21 @@ def add_flag_to_cmd(flag, val): separator = "" if no_space else " " return "{}{}{}".format(flag, separator, str(val).strip()) + + +def is_comp_standalone(case): + """ + Test if the case is a single component standalone + such as FKESSLER + """ + stubcnt = 0 + classes = case.get_values("COMP_CLASSES") + for comp in classes: + if case.get_value("COMP_{}".format(comp)) == "s{}".format(comp.lower()): + stubcnt = stubcnt + 1 + else: + model = comp.lower() + numclasses = len(classes) + if stubcnt >= numclasses - 2: + return True, model + return False, get_model() diff --git a/CMakeLists.txt b/CMakeLists.txt index 94546b8e405..b8de549f4ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,73 +22,6 @@ include_directories(${NetCDF_C_INCLUDE_DIRS} ${NetCDF_Fortran_INCLUDE_DIRS}) # TODO: Some of the below should be done in the relevant directories, not in # this top level CMakeLists. -# ------------------------------------------------------------------------ -# Build mct -# ------------------------------------------------------------------------ -if (EXISTS ${SRC_ROOT}/libraries/mct) - set(MCT_ROOT "${SRC_ROOT}/libraries/mct") -else() - set(MCT_ROOT "${SRC_ROOT}/externals/mct") -endif() - -if (USE_MPI_SERIAL) - set(ENABLE_MPI_SERIAL "--enable-mpiserial") -else() - set(ENABLE_MPI_SERIAL "") -endif() - -ExternalProject_add(mct_project - PREFIX ${CMAKE_CURRENT_BINARY_DIR} - SOURCE_DIR ${MCT_ROOT} - BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/mct - CONFIGURE_COMMAND ${MCT_ROOT}/configure ${ENABLE_MPI_SERIAL} --enable-debugging --prefix=${CMAKE_CURRENT_BINARY_DIR} CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} CFLAGS=${CFLAGS} FCFLAGS=${FFLAGS} SRCDIR=${MCT_ROOT} DEBUG="-g" - BUILD_COMMAND $(MAKE) SRCDIR=${MCT_ROOT} - # Leave things in rather than "installing", because we have - # no need to move things around inside of the CMake binary directory. Also, - # mpi-serial doesn't install properly in the out-of-source build - INSTALL_COMMAND : - ) -# This copy_makefiles step is needed because mct currently doesn't support an -# out-of-source build. I am replicating what is done for the CIME system build. -ExternalProject_add_step(mct_project copy_makefiles - DEPENDEES configure - DEPENDERS build - WORKING_DIRECTORY - COMMAND cp -p /Makefile . - COMMAND mkdir -p mct - COMMAND cp -p /mct/Makefile mct/ - COMMAND mkdir -p mpeu - COMMAND cp -p /mpeu/Makefile mpeu/ - ) -if (USE_MPI_SERIAL) - ExternalProject_add_step(mct_project copy_mpi_serial_files - DEPENDEES configure - DEPENDERS build - WORKING_DIRECTORY - COMMAND mkdir -p mpi-serial - COMMAND cp -p /mpi-serial/Makefile mpi-serial/ - COMMAND cp /mpi-serial/mpif.h mpi-serial/ - COMMAND cp /mpi-serial/mpi.h mpi-serial/ - ) -endif() - -# Tell cmake to look for libraries & mod files here, because this is where we built libraries -include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct) -include_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) -link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mct) -link_directories(${CMAKE_CURRENT_BINARY_DIR}/mct/mpeu) -if (USE_MPI_SERIAL) - # We need to list the mpi-serial include directory before system-level - # directories so that we're sure to use mpi-serial's mpif.h instead of - # an mpif.h from a system path. - include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial) - link_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/mct/mpi-serial) -endif() - -# ------------------------------------------------------------------------ -# Done MCT build -# ------------------------------------------------------------------------ - # Now a bunch of includes for share code. # csm_share (we don't build it here because it seems to be built differently @@ -96,7 +29,6 @@ endif() if (EXISTS ${SRC_ROOT}/share/src) add_subdirectory(${SRC_ROOT}/share/src share_src) - add_subdirectory(${SRC_ROOT}/components/cpl7/mct_shr mct_src) add_subdirectory(${SRC_ROOT}/share/unit_test_stubs/util csm_share_stubs) include_directories(${SRC_ROOT}/share/include) else() @@ -115,9 +47,4 @@ else() endif() # Now the actual test directories. -if (EXISTS ${SRC_ROOT}/components/cpl7/driver/unit_test) - add_subdirectory(${SRC_ROOT}/components/cpl7/driver/unit_test unit_test) -else() - add_subdirectory(${SRC_ROOT}/driver-mct/unit_test unit_test) -endif() add_subdirectory(${SRC_ROOT}/share/test/unit ${CMAKE_BINARY_DIR}/unittests) diff --git a/Externals.cfg b/Externals.cfg index 01537b54197..5efcc075503 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,19 +1,19 @@ [ccs_config] -tag = ccs_config_cesm0.0.76 +tag = ccs_config_cesm0.0.91 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm local_path = ccs_config required = True [cmeps] -tag = cmeps0.14.38 +tag = cmeps0.14.49 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.19 +tag = cdeps1.0.26 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -21,14 +21,14 @@ externals = Externals_CDEPS.cfg required = True [cpl7] -tag = cpl77.0.6 +tag = cpl77.0.8 protocol = git repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps local_path = components/cpl7 required = True [share] -tag = share1.0.17 +tag = share1.0.18 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share @@ -42,7 +42,7 @@ local_path = libraries/mct required = True [parallelio] -tag = pio2_6_0 +tag = pio2_6_2 protocol = git repo_url = https://github.com/NCAR/ParallelIO local_path = libraries/parallelio diff --git a/Externals_cime.cfg b/Externals_cime.cfg new file mode 100644 index 00000000000..04d7306e241 --- /dev/null +++ b/Externals_cime.cfg @@ -0,0 +1,7 @@ +[CIME/non_py/cprnc] +protocol = git +from_submodule=True +required = True + +[externals_description] +schema_version = 1.0.0 diff --git a/doc/source/users_guide/testing.rst b/doc/source/users_guide/testing.rst index 40868d2bbdd..061c62e3152 100644 --- a/doc/source/users_guide/testing.rst +++ b/doc/source/users_guide/testing.rst @@ -441,10 +441,12 @@ The following pseudo code is an example of this customization.:: ------- str Storing throughput value. + str + Open baseline file for writing. """ current = analyze_throughput(...) - return json.dumps(current) + return json.dumps(current), "w" def perf_get_memory(case): """ @@ -457,10 +459,12 @@ The following pseudo code is an example of this customization.:: ------- str Storing memory value. + str + Open baseline file for writing. """ current = analyze_memory(case) - return json.dumps(current) + return json.dumps(current), "w" def perf_compare_throughput_baseline(case, baseline, tolerance): """ diff --git a/docker/config_machines.xml b/docker/.cime/config_machines.v2.xml similarity index 96% rename from docker/config_machines.xml rename to docker/.cime/config_machines.v2.xml index ea60a2cfbb3..242150d750c 100644 --- a/docker/config_machines.xml +++ b/docker/.cime/config_machines.v2.xml @@ -3,7 +3,7 @@ Docker - docker + LINUX gnu,gnuX diff --git a/docker/.cime/config_machines.v3.xml b/docker/.cime/config_machines.v3.xml new file mode 100644 index 00000000000..98a0cba3f66 --- /dev/null +++ b/docker/.cime/config_machines.v3.xml @@ -0,0 +1,7 @@ + + + + + docker + + diff --git a/docker/docker.cmake b/docker/.cime/docker.cmake similarity index 100% rename from docker/docker.cmake rename to docker/.cime/docker.cmake diff --git a/docker/.cime/docker/config_machines.xml b/docker/.cime/docker/config_machines.xml new file mode 100644 index 00000000000..e15fd7eaa49 --- /dev/null +++ b/docker/.cime/docker/config_machines.xml @@ -0,0 +1,39 @@ + + Docker + LINUX + + gnu,gnuX + openmpi + CIME + /storage/timings + CIME + /storage/cases + /storage/inputdata + /storage/inputdata-clmforc + /storage/archive/$CASE + /storage/baselines/$COMPILER + /storage/tools/cprnc + make + 4 + e3sm_developer + none + boutte3@llnl.gov + 8 + 8 + + mpiexec + + -n {{ total_tasks }} + --oversubscribe + + + + $CASEROOT/run + $CASEROOT/bld + + 1 + 1 + /opt/conda + /opt/conda + + diff --git a/docker/Dockerfile b/docker/Dockerfile index 6bcee0d1e0f..a148d921d4c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,31 +1,13 @@ ARG MAMBAFORGE_VERSION=4.14.0-0 FROM condaforge/mambaforge:${MAMBAFORGE_VERSION} AS base -ARG PNETCDF_VERSION=1.12.3 -ENV PNETCDF_VERSION=${PNETCDF_VERSION} - -ARG LIBNETCDF_VERSION=4.9.1 -ENV LIBNETCDF_VERSION=${LIBNETCDF_VERSION} - -ARG NETCDF_FORTRAN_VERSION=* -ENV NETCDF_FORTRAN_VERSION=${NETCDF_FORTRAN_VERSION} - -ARG ESMF_VERSION=* -ENV ESMF_VERSION=${ESMF_VERSION} - -ARG GCC_VERSION=10.* -ENV GCC_VERSION=${GCC_VERSION} - -ENV USER=root -ENV LOGNAME=root - SHELL ["/bin/bash", "-c"] # First layer as they never change, required for E3SM testing, TODO: fix in unittesting as well -RUN mkdir -p /cache/cpl/gridmaps/oQU240 /cache/share/domains && \ - wget -O /cache/cpl/gridmaps/oQU240/map_oQU240_to_ne4np4_aave.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/cpl/gridmaps/oQU240/map_oQU240_to_ne4np4_aave.160614.nc && \ - wget -O /cache/share/domains/domain.ocn.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc && \ - wget -O /cache/share/domains/domain.lnd.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc +RUN mkdir -p /storage/inputdata/cpl/gridmaps/oQU240 /storage/inputdata/share/domains && \ + wget -O /storage/inputdata/cpl/gridmaps/oQU240/map_oQU240_to_ne4np4_aave.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/cpl/gridmaps/oQU240/map_oQU240_to_ne4np4_aave.160614.nc && \ + wget -O /storage/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.ocn.ne4np4_oQU240.160614.nc && \ + wget -O /storage/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc https://portal.nersc.gov/project/e3sm/inputdata/share/domains/domain.lnd.ne4np4_oQU240.160614.nc # Install common packages RUN mamba install --yes -c conda-forge \ @@ -43,6 +25,16 @@ RUN mamba install --yes -c conda-forge \ openssh && \ rm -rf /opt/conda/pkgs/* +# Compilers and libraries +ARG LIBNETCDF_VERSION=4.9.1 +ENV LIBNETCDF_VERSION=${LIBNETCDF_VERSION} +ARG NETCDF_FORTRAN_VERSION=* +ENV NETCDF_FORTRAN_VERSION=${NETCDF_FORTRAN_VERSION} +ARG ESMF_VERSION=* +ENV ESMF_VERSION=${ESMF_VERSION} +ARG GCC_VERSION=10.* +ENV GCC_VERSION=${GCC_VERSION} + # Install version locked packages # gcc, gxx, gfortran provide symlinks for x86_64-conda-linux-gnu-* # ar and ranlib are not symlinked @@ -61,10 +53,11 @@ RUN mamba install --yes -c conda-forge \ gfortran && \ rm -rf /opt/conda/pkgs/* && \ ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ar /opt/conda/bin/ar && \ - ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ranlib /opt/conda/bin/ranlib + ln -sf /opt/conda/bin/x86_64-conda-linux-gnu-ranlib /opt/conda/bin/ranlib && \ + cpan install XML::LibXML Switch -# Install cpan packages -RUN cpan install XML::LibXML Switch +ARG PNETCDF_VERSION=1.12.3 +ENV PNETCDF_VERSION=${PNETCDF_VERSION} # Build pnetcdf RUN curl -L -k -o "${PWD}/pnetcdf.tar.gz" \ @@ -84,10 +77,38 @@ RUN curl -L -k -o "${PWD}/pnetcdf.tar.gz" \ make install && \ rm -rf "${PWD}/pnetcdf" -RUN mkdir /root/.cime +# CESM dependencies +ENV CCS_CONFIG_TAG=ccs_config_cesm0.0.88 +ENV CMEPS_TAG=cmeps0.14.47 +ENV CDEPS_TAG=cdeps1.0.26 +ENV CPL7_TAG=cpl77.0.8 +ENV SHARE_TAG=share1.0.18 +ENV MCT_TAG=MCT_2.11.0 +ENV PARALLELIO_TAG=pio2_6_2 + +RUN git clone -b ${CCS_CONFIG_TAG} https://github.com/ESMCI/ccs_config_cesm /src/ccs_config && \ + git clone -b ${CMEPS_TAG} https://github.com/ESCOMP/CMEPS.git /src/components/cmeps && \ + git clone -b ${CDEPS_TAG} https://github.com/ESCOMP/CDEPS.git /src/components/cdeps && \ + git clone -b ${CPL7_TAG} https://github.com/ESCOMP/CESM_CPL7andDataComps /src/components/cpl7 && \ + git clone -b ${SHARE_TAG} https://github.com/ESCOMP/CESM_share /src/share && \ + git clone -b ${MCT_TAG} https://github.com/MCSclimate/MCT /src/libraries/mct && \ + git clone -b ${PARALLELIO_TAG} https://github.com/NCAR/ParallelIO /src/libraries/parallelio && \ + mkdir -p /storage/timings + +ARG CIME_BRANCH=master +ARG CIME_REPO=https://github.com/esmci/cime + +# Separate layer, it's most likely to change +RUN git clone -b ${CIME_BRANCH} ${CIME_REPO} /src/cime + +# General variables +ENV USER=root +ENV LOGNAME=container +ENV ESMFMKFILE=/opt/conda/lib/esmf.mk + +WORKDIR /src/cime -COPY config_machines.xml /root/.cime/ -COPY docker.cmake /root/.cime/ +COPY .cime /root/.cime COPY entrypoint.sh /entrypoint.sh ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index c89d6a9d375..dd0ac467233 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,176 +1,114 @@ #!/bin/bash -set -x - -readonly INIT=${INIT:-"true"} -readonly UPDATE_CIME=${UPDATE_CIME:-"false"} -readonly GIT_SHALLOW=${GIT_SHALLOW:-"false"} - -declare -xr CIME_REPO=${CIME_REPO:-https://github.com/ESMCI/cime} -declare -xr E3SM_REPO=${E3SM_REPO:-https://github.com/E3SM-Project/E3SM} -declare -xr CESM_REPO=${CESM_REPO:-https://github.com/ESCOMP/CESM} - -####################################### -# Clones git repository -####################################### -function clone_repo() { - local repo="${1}" - local path="${2}" - local branch="${3}" - local extras="" - - if [[ "${GIT_SHALLOW}" == "true" ]] - then - extras="${extras} --depth 1" - fi - - git clone -b "${branch}" ${extras} "${repo}" "${path}" || true -} +DEBUG="${DEBUG:-false}" +SRC_PATH="${SRC_PATH:-`pwd`}" +# Treeless clone +GIT_FLAGS="${GIT_FLAGS:---filter=tree:0}" +# Shallow submodule checkout +GIT_SUBMODULE_FLAGS="${GIT_SUBMODULE_FLAGS:---recommend-shallow}" + +echo "DEBUG = ${DEBUG}" +echo "SRC_PATH = ${SRC_PATH}" +echo "GIT_FLAGS = ${GIT_FLAGS}" +echo "GIT_SUBMODULE_FLAGS = ${GIT_SUBMODULE_FLAGS}" + +if [[ "$(echo ${DEBUG} | tr -s '[:upper:]' '[:lower:]')" == "true" ]] +then + set -x +fi ####################################### # Fixes mct/mpeu to use ARFLAGS environment variable # # TODO need to make an offical PR this is temporary. ####################################### -function fixup_mct { +function fix_mct_arflags { local mct_path="${1}" # TODO make PR to fix if [[ ! -e "${mct_path}/mct/Makefile.bak" ]] then + echo "Fixing AR variable in ${mct_path}/mct/Makefile" + sed -i".bak" "s/\$(AR)/\$(AR) \$(ARFLAGS)/g" "${mct_path}/mct/Makefile" fi if [[ ! -e "${mct_path}/mpeu/Makefile.bak" ]] then + echo "Fixing AR variable in ${mct_path}/mpeu/Makefile" + sed -i".bak" "s/\$(AR)/\$(AR) \$(ARFLAGS)/g" "${mct_path}/mpeu/Makefile" fi } ####################################### +# Fixes gitmodules to use https rather than ssh ####################################### -function update_cime() { - local path="${1}" - - if [[ "${UPDATE_CIME}" == "true" ]] - then - echo "Updating CIME using repository ${CIME_REPO} and branch ${CIME_BRANCH}" - - pushd "${path}" - - git remote set-url origin "${CIME_REPO}" - - if [[ "${GIT_SHALLOW}" == "true" ]] - then - git remote set-branches origin "*" - fi - - git fetch origin - - git checkout "${CIME_BRANCH:-master}" - - popd - fi +function fix_gitmodules() { + sed -i".bak" "s/git@github.com:/https:\/\/github.com\//g" "${1}/.gitmodules" } -####################################### -# Creates an environment with E3SM source. -####################################### -function init_e3sm() { - export CIME_MODEL="e3sm" - - local extras="" - local install_path="${INSTALL_PATH:-/src/E3SM}" - local cache_path="${cache_path:-/storage/inputdata}" - - if [[ ! -e "${install_path}" ]] - then - clone_repo "${E3SM_REPO}" "${install_path}" "${E3SM_BRANCH:-master}" - - cd "${install_path}" - - if [[ ! -e "${PWD}/.gitmodules.bak" ]] - then - sed -i".bak" "s/git@github.com:/https:\/\/github.com\//g" "${PWD}/.gitmodules" - fi - - if [[ "${GIT_SHALLOW}" == "true" ]] - then - extras=" --depth 1" - fi - - git submodule update --init ${extras} - fi - - fixup_mct "${install_path}/externals/mct" - - update_cime "${install_path}/cime" +if [[ "${CIME_MODEL}" == "e3sm" ]] +then + echo "Setting up E3SM" - mkdir -p /storage/inputdata + [[ ! -e "${SRC_PATH}/E3SM" ]] && git clone -b ${E3SM_BRANCH:-master} ${GIT_FLAGS} ${E3SM_REPO:-https://github.com/E3SM-Project/E3SM} "${SRC_PATH}/E3SM" - rsync -vr /cache/ /storage/inputdata/ + pushd "${SRC_PATH}/E3SM" - cd "${install_path}/cime" -} + git config --global --add safe.directory "${PWD}" -####################################### -# Creates an environment with CESM source. -####################################### -function init_cesm() { - export CIME_MODEL="cesm" + # fix E3SM gitmodules + fix_gitmodules "${PWD}" - local install_path="${INSTALL_PATH:-/src/CESM}" - - if [[ ! -e "${install_path}" ]] - then - clone_repo "${CESM_REPO}" "${install_path}" "${CESM_BRANCH:-master}" - fi + git status - cd "${install_path}" + # checkout submodules + git submodule update --init "${GIT_SUBMODULE_FLAGS}" - "${install_path}/manage_externals/checkout_externals" + # fix mct arflags flags + fix_mct_arflags "${SRC_PATH}/E3SM/externals/mct" - fixup_mct "${install_path}/libraries/mct" + pushd cime - update_cime "${install_path}/cime/" + # fix CIME gitmodules + fix_gitmodules "${PWD}" - cd "${install_path}/cime" -} + git config --global --add safe.directory "${PWD}" + git config --global --add safe.directory "${PWD}/CIME/non_py/cprnc" -####################################### -# Creates an environment with minimal model requirements. -# Similar to old github actions environment. -####################################### -function init_cime() { - export CIME_MODEL="cesm" - export ESMFMKFILE="/opt/conda/lib/esmf.mk" + # checkout submodules + git submodule update --init "${GIT_SUBMODULE_FLAGS}" - local install_path="${INSTALL_PATH:-/src/cime}" + # link v2 config_machines + ln -sf /root/.cime/config_machines.v2.xml /root/.cime/config_machines.xml +elif [[ "${CIME_MODEL}" == "cesm" ]] +then + echo "Setting up CESM" - if [[ ! -e "${install_path}" ]] + # copy pre cloned repos to new source path + if [[ "${SRC_PATH}" != "/src/cime" ]] then - clone_repo "${CIME_REPO}" "${install_path}" "${CIME_BRANCH:-master}" + cp -rf /src/ccs_config /src/components /src/libraries /src/share "${SRC_PATH}/../" fi - # required to using checkout_externals script - clone_repo "${CESM_REPO}" "/src/CESM" "${CESM_BRANCH:-master}" + git config --global --add safe.directory "${PWD}" + git config --global --add safe.directory "${PWD}/CIME/non_py/cprnc" - cd "${install_path}" + # fix CIME gitmodules + fix_gitmodules "${PWD}" - "/src/CESM/manage_externals/checkout_externals" + # update CIME submodules + git submodule update --init "${GIT_SUBMODULE_FLAGS}" - fixup_mct "${install_path}/libraries/mct" - - update_cime "${install_path}" - - cd "${install_path}" -} + # fix mct argflags + fix_mct_arflags /src/libraries/mct -if [[ ! -e "${HOME}/.cime" ]] -then - ln -sf "/root/.cime" "${HOME}/.cime" + # link v3 config_machines + ln -sf /root/.cime/config_machines.v3.xml /root/.cime/config_machines.xml fi +# load batch specific entrypoint if [[ -e "/entrypoint_batch.sh" ]] then echo "Sourcing batch entrypoint" @@ -178,17 +116,4 @@ then . "/entrypoint_batch.sh" fi -if [[ "${INIT}" == "true" ]] -then - if [[ "${CIME_MODEL}" == "e3sm" ]] - then - init_e3sm - elif [[ "${CIME_MODEL}" == "cesm" ]] - then - init_cesm - else - init_cime - fi - - exec "${@}" -fi +exec "${@}" diff --git a/tools/mapping/gen_domain_files/src/Makefile b/tools/mapping/gen_domain_files/src/Makefile index 5a12b7daec5..a6fa89bf9bc 100644 --- a/tools/mapping/gen_domain_files/src/Makefile +++ b/tools/mapping/gen_domain_files/src/Makefile @@ -122,7 +122,7 @@ OBJS := gen_domain.o # Append user defined compiler and load flags to Makefile defaults CFLAGS += $(USER_CFLAGS) -I$(INC_NETCDF) -FFLAGS += $(USER_FFLAGS) -I$(MOD_NETCDF) -I$(INC_NETCDF) +FFLAGS += $(USER_FFLAGS) -I$(MOD_NETCDF) -I$(INC_NETCDF) $(CMAKE_Fortran_FLAGS) LDFLAGS += $(USER_LDFLAGS) # Set user specified linker diff --git a/tools/mapping/gen_domain_files/test_gen_domain.sh b/tools/mapping/gen_domain_files/test_gen_domain.sh index b80d482ce13..06e995a8c62 100755 --- a/tools/mapping/gen_domain_files/test_gen_domain.sh +++ b/tools/mapping/gen_domain_files/test_gen_domain.sh @@ -81,12 +81,11 @@ fi # Build the cprnc executable (for comparison of netcdf files) echo "" >> ${test_log} echo "Building cprnc in ${PWD}/builds ..." >> ${test_log} -cp ${cime_root}/CIME/non_py/cprnc/*.F90 . -cp ${cime_root}/CIME/non_py/cprnc/Makefile . -cp ${cime_root}/CIME/non_py/cprnc/Depends . -cp ${cime_root}/CIME/non_py/cprnc/*.in . -(. .env_mach_specific.sh && make GENF90=${cime_root}/CIME/non_py/externals/genf90/genf90.pl) >> ${test_log} 2>&1 -if [ ! -f cprnc ]; then +mkdir ${PWD}/builds/cprnc +cd ${PWD}/builds/cprnc +cmake -DCMAKE_INSTALL_PREFIX=${PWD} ${cime_root}/CIME/non_py/cprnc +make install +if [ ! -f bin/cprnc ]; then echo "ERROR building cprnc" >&2 echo "cat ${test_log} for more info" >&2 exit 1 @@ -104,33 +103,33 @@ for baseline in ${ocn_baseline} ${lnd_baseline}; do # and adding in datestring for current day and .nc file extension. testfile=`basename ${baseline} | rev | cut -d. -f3- | rev`.${datestring}.nc if [ ! -f ${testfile} ]; then - echo "ERROR: ${testfile} not generated" >&2 - echo "cat ${test_log} for more info" >&2 - exit 1 + echo "ERROR: ${testfile} not generated" >&2 + echo "cat ${test_log} for more info" >&2 + exit 1 fi # Compare against baseline and print report from cprnc comparison echo "Comparing $testfile against ${baseline}..." - (. builds/.env_mach_specific.sh && ./builds/cprnc -m ${testfile} ${baseline}) >> ${test_log} 2>&1 + (. builds/.env_mach_specific.sh && ./builds/bin/cprnc -m ${testfile} ${baseline}) >> ${test_log} 2>&1 # Check results last=`tail -n3 ${test_log}` if [[ ${last} =~ "STOP" ]]; then - echo ${last} >&2 - echo "Error running cprnc" >&2 - echo "cat ${test_log} for more info" >&2 - exit 1 + echo ${last} >&2 + echo "Error running cprnc" >&2 + echo "cat ${test_log} for more info" >&2 + exit 1 fi if [[ ${last} =~ "DIFFERENT" ]]; then - echo ${last} >&2 - echo ${baseline} DIFFERENT FROM ${testfile} >&2 - echo "cat ${test_log} for more info" >&2 - exit 1 + echo ${last} >&2 + echo ${baseline} DIFFERENT FROM ${testfile} >&2 + echo "cat ${test_log} for more info" >&2 + exit 1 fi if ! [[ ${last} =~ "IDENTICAL" ]]; then - echo ${last} >&2 - echo "undetermined output from cprnc" >&2 - echo "cat ${test_log} for more info" >&2 - exit 1 + echo ${last} >&2 + echo "undetermined output from cprnc" >&2 + echo "cat ${test_log} for more info" >&2 + exit 1 fi done