From 4ae7d6fe5ba148db30cbbccb4327d986ca3d330d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 21 Feb 2023 15:48:25 -0700 Subject: [PATCH 01/18] include PRISM precipitation data streams --- cime_config/stream_cdeps.py | 1 + datm/cime_config/namelist_definition_datm.xml | 2 +- datm/cime_config/stream_definition_datm.xml | 40 ++++++++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cime_config/stream_cdeps.py b/cime_config/stream_cdeps.py index c989532e..67967077 100644 --- a/cime_config/stream_cdeps.py +++ b/cime_config/stream_cdeps.py @@ -95,6 +95,7 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file # endif # end while index += 1 + line = case.get_resolved_value(line) lines_input_new.append(line) #end while diff --git a/datm/cime_config/namelist_definition_datm.xml b/datm/cime_config/namelist_definition_datm.xml index c677bffe..355b39f2 100644 --- a/datm/cime_config/namelist_definition_datm.xml +++ b/datm/cime_config/namelist_definition_datm.xml @@ -38,7 +38,7 @@ 1PT.urbanc_alpha - NEON.$NEONSITE + NEON.$NEONSITE,NEON.$NEONSITE.PRECIP CLM_USRDAT.$CLM_USRDAT_NAME diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index c29decdc..6f4e48e3 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -236,7 +236,7 @@ $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc - PRECTmms Faxa_precn + FSDS Faxa_swdn ZBOT Sa_z TBOT Sa_tbot @@ -267,6 +267,44 @@ single + + + + + + + + none + + + /glade/u/home/tking/neon/prism_$NEONSITE_%y.nc + + + PRECIP Faxa_precn + + null + + none + + null + $DATM_YR_ALIGN + $DATM_YR_START + $DATM_YR_END + 0 + + linear + + + cycle + limit + + + 1.5 + + single + + + From 400f475b3233303e713821db0a15f35dfc684032 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 7 Mar 2023 09:58:12 -0700 Subject: [PATCH 02/18] changed name of PRISM PRECIP stream --- datm/cime_config/namelist_definition_datm.xml | 2 +- datm/cime_config/stream_definition_datm.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datm/cime_config/namelist_definition_datm.xml b/datm/cime_config/namelist_definition_datm.xml index 355b39f2..d59156e1 100644 --- a/datm/cime_config/namelist_definition_datm.xml +++ b/datm/cime_config/namelist_definition_datm.xml @@ -38,7 +38,7 @@ 1PT.urbanc_alpha - NEON.$NEONSITE,NEON.$NEONSITE.PRECIP + NEON.$NEONSITE,NEON.PRISM_PRECIP.$NEONSITE CLM_USRDAT.$CLM_USRDAT_NAME diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 6f4e48e3..1e95074c 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -269,10 +269,10 @@ - + - + none From 783302437af8a14b27dab0189b0a7060a9530b83 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 22 Mar 2023 12:23:37 -0600 Subject: [PATCH 03/18] temporary updates for nl and stream changes --- cime_config/stream_cdeps.py | 5 ++++- datm/cime_config/buildnml | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cime_config/stream_cdeps.py b/cime_config/stream_cdeps.py index 67967077..f32b294a 100644 --- a/cime_config/stream_cdeps.py +++ b/cime_config/stream_cdeps.py @@ -135,7 +135,10 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file # write contents of stream file for stream_name in stream_names: - if stream_name.startswith("NEON."): + if stream_name.startswith("NEON.PRISM"): + self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.PRISM_PRECIP.$NEONSITE"}, + err_msg="No stream_entry {} found".format(stream_name)) + elif stream_name.startswith("NEON."): self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.$NEONSITE"}, err_msg="No stream_entry {} found".format(stream_name)) diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index 575949b1..de9afac7 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -42,6 +42,7 @@ def _get_neon_data_availability(case, neonsite): if not os.path.exists(fullpath): inputdata = Inputdata() protocol, address, user, passwd, _, _, inventory = inputdata.get_next_server(attributes={'CLM_USRDAT_NAME':'NEON'}) + # CLM_USRDAT_NAME could also be NEON.PRISM success = case.check_input_data(protocol=protocol, address=address, download=True) logger.debug("_get_neon_data_availablity: address {} inventory is {} success {}".format(address, inventory, success)) @@ -106,6 +107,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path datm_presaero = case.get_value("DATM_PRESAERO") datm_presndep = case.get_value("DATM_PRESNDEP") datm_preso3 = case.get_value("DATM_PRESO3") + clm_usrdat_name = case.get_value("CLM_USRDAT_NAME") datm_co2_tseries = case.get_value("DATM_CO2_TSERIES") atm_grid = case.get_value("ATM_GRID") model_grid = case.get_value("GRID") @@ -129,6 +131,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path logger.debug("DATM presndep mode is {}".format(datm_presndep)) logger.debug("DATM preso3 mode is {}".format(datm_preso3)) logger.debug("DATM topo mode is {}".format(datm_topo)) + logger.debug("CLM_USRDAT_NAME is {}".format(clm_usrdat_name)) # Initialize namelist defaults config = {} @@ -148,6 +151,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path config['datm_presaero'] = datm_presaero config['datm_presndep'] = datm_presndep config['datm_preso3'] = datm_preso3 + config['clm_usrdat_name'] = clm_usrdat_name if case.get_value('PTS_LON'): scol_lon = float(case.get_value('PTS_LON')) @@ -184,6 +188,11 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path streamlist.append("topo.{}".format(datm_topo)) if datm_co2_tseries != "none": streamlist.append("co2tseries.{}".format(datm_co2_tseries)) + if 'NEON' in clm_usrdat_name: + # streamlist.append("NEON.PRISM_PRECIP.CPER") + # streamlist.append("NEON.PRISM_PRECIP.".format(clm_usrdat_name[-4:])) # this ends up being .RISM because the last for of NEON.PRISM are RISM + # streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) # USE THIS VERSION ONCE IT DOESN'T CAUSE SVN FAILURES + streamlist.append("NEON.PRISM_PRECIP."+neonsite) bias_correct = nmlgen.get_value("bias_correct") if bias_correct is not None: From f0942b9693dfe9d6ade9bdde31e8e5d44165c535 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 27 Mar 2023 16:19:14 -0600 Subject: [PATCH 04/18] update buildnml --- cime_config/buildnml | 330 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100755 cime_config/buildnml diff --git a/cime_config/buildnml b/cime_config/buildnml new file mode 100755 index 00000000..2e915897 --- /dev/null +++ b/cime_config/buildnml @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 + +"""Namelist creator for CDEPS data atm model. +""" + +# Typically ignore this. +# pylint: disable=invalid-name + +# Disable these because this is our standard setup +# pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position + +import os, sys + +_CDEPS_CONFIG = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,os.pardir,"cime_config") +_CIMEROOT = os.environ.get("CIMEROOT") +if _CIMEROOT is None: + raise SystemExit("ERROR: must set CIMEROOT environment variable") +_LIBDIR = os.path.join(_CIMEROOT, "CIME", "Tools") +sys.path.append(_LIBDIR) +sys.path.append(_CDEPS_CONFIG) + +from datetime import datetime +from standard_script_setup import * +from CIME.case import Case +from CIME.nmlgen import NamelistGenerator +from CIME.utils import expect, safe_copy, SharedArea +from CIME.buildnml import create_namelist_infile, parse_input, copy_inputs_to_rundir +from CIME.XML.files import Files +from CIME.XML.inputdata import Inputdata +from stream_cdeps import StreamCDEPS + +logger = logging.getLogger(__name__) + +def _get_neon_data_availability(case, neonsite): + """ + Read the NEON inventory file and check availablity of tower data for this case. + Put available data in datavaliddate list and return it + Currently the valid date information is not used + """ + rundir = case.get_value("RUNDIR") + fullpath = os.path.join(rundir,"listing.csv") + if not os.path.exists(fullpath): + inputdata = Inputdata() + protocol, address, user, passwd, _, _, inventory = inputdata.get_next_server(attributes={'CLM_USRDAT_NAME':'NEON'}) + # CLM_USRDAT_NAME could also be NEON.PRISM + success = case.check_input_data(protocol=protocol, address=address, download=True) + logger.debug("_get_neon_data_availablity: address {} inventory is {} success {}".format(address, inventory, success)) + + server = CIME.Servers.WGET.wget_login(address, user, passwd) + if server: + with SharedArea(): + success = server.getfile(inventory, fullpath) + expect(success, "Could not download NEON data listing file from server") + + expect(os.path.exists(fullpath), "Could not download NEON data listing file from server") + datavaliddate = [] + oldestdate = None + newestdate = None + neonatm = None + dataversion = case.get_value("NEONVERSION") + if dataversion == "latest": + dataversions = ["v2", "v1"] + else: + dataversions = [dataversion] + + for version in dataversions: + if not newestdate: + with open(fullpath, "r") as fd: + for line in fd.readlines(): + fpath, fname = os.path.split(line) + if fpath.endswith(f'cdeps/{version}/{neonsite}'): + neonatm, date, _ = line.strip().split(',') + if neonatm.endswith('.nc'): + neonatm = neonatm.split('/')[-1] + newestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc") + if not oldestdate: + oldestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc") + neonatm = f'cdeps/{version}/{neonsite}/'+neonatm + datavaliddate.append(neonatm) + if newestdate: + logger.info("Found tower data version {} for {} through {}".format(version, oldestdate, newestdate)) + datavaliddate.sort() + return datavaliddate + + expect(newestdate, "No tower data found on server for NEON site {}".format(neonsite)) + return None + +# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements +#################################################################################### +def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path): +#################################################################################### + """Write out the namelist for this component. + + Most arguments are the same as those for `NamelistGenerator`. The + `inst_string` argument is used as a suffix to distinguish files for + different instances. The `confdir` argument is used to specify the directory + in which output files will be placed. + """ + + #---------------------------------------------------- + # Write out datm_in and datm.streams.xml + #---------------------------------------------------- + + caseroot = case.get_value("CASEROOT") + datm_mode = case.get_value("DATM_MODE") + datm_topo = case.get_value("DATM_TOPO") + datm_presaero = case.get_value("DATM_PRESAERO") + datm_presndep = case.get_value("DATM_PRESNDEP") + datm_preso3 = case.get_value("DATM_PRESO3") + clm_usrdat_name = case.get_value("CLM_USRDAT_NAME") + datm_co2_tseries = case.get_value("DATM_CO2_TSERIES") + atm_grid = case.get_value("ATM_GRID") + model_grid = case.get_value("GRID") + comp_lnd = case.get_value("COMP_LND") + + # Check for incompatible options. + if "CLM" in datm_mode and comp_lnd == "clm": + expect(datm_presaero != "none", + "A DATM_MODE for CLM is incompatible with DATM_PRESAERO=none.") + expect(datm_presndep != "none", + "A DATM_MODE for CLM is incompatible with DATM_PRESNDEP=none.") + expect(datm_preso3 != "none", + "A DATM_MODE for CLM is incompatible with DATM_PRESO3=none.") + expect(datm_topo != "none", + "A DATM_MODE for CLM is incompatible with DATM_TOPO=none.") + + # Log some settings. + logger.debug("DATM mode is {}".format(datm_mode)) + logger.debug("DATM grid is {}".format(atm_grid)) + logger.debug("DATM presaero mode is {}".format(datm_presaero)) + logger.debug("DATM presndep mode is {}".format(datm_presndep)) + logger.debug("DATM preso3 mode is {}".format(datm_preso3)) + logger.debug("DATM topo mode is {}".format(datm_topo)) + logger.debug("CLM_USRDAT_NAME is {}".format(clm_usrdat_name)) + + # Initialize namelist defaults + config = {} + available_neon_data = None + if 'CLM_USRDAT' in model_grid: + config['model_grid'] = 'CLM_USRDAT' + neonsite = case.get_value("NEONSITE") + if neonsite: + config['neon'] = "True" + # download and use the listing.csv file to determine data availablity + available_neon_data = _get_neon_data_availability(case, neonsite) + else: + config['model_grid'] = model_grid + + config['datm_mode'] = datm_mode + config['datm_co2_tseries'] = datm_co2_tseries + config['datm_presaero'] = datm_presaero + config['datm_presndep'] = datm_presndep + config['datm_preso3'] = datm_preso3 + config['clm_usrdat_name'] = clm_usrdat_name + + if case.get_value('PTS_LON'): + scol_lon = float(case.get_value('PTS_LON')) + else: + scol_lon = -999. + if case.get_value('PTS_LAT'): + scol_lat = float(case.get_value('PTS_LAT')) + else: + scol_lat = -999. + if case.get_value('ATM_DOMAIN_FILE'): + if scol_lon > -999. and scol_lat > -999. and case.get_value("ATM_DOMAIN_FILE") != "UNSET": + config['single_column'] = 'true' + else: + config['single_column'] = 'false' + + nmlgen.init_defaults(infile, config) + + # Generate datm_in + namelist_file = os.path.join(confdir, "datm_in") + nmlgen.write_output_file(namelist_file, data_list_path, groups=['datm_nml']) + + # Determine streams + streamlist = nmlgen.get_streams() + if type(streamlist) == type(str()): + streamlist = [] + + if datm_presaero != "none": + streamlist.append("presaero.{}".format(datm_presaero)) + if datm_presndep != "none": + streamlist.append("presndep.{}".format(datm_presndep)) + if datm_preso3 != "none": + streamlist.append("preso3.{}".format(datm_preso3)) + if datm_topo != "none": + streamlist.append("topo.{}".format(datm_topo)) + if datm_co2_tseries != "none": + streamlist.append("co2tseries.{}".format(datm_co2_tseries)) + if 'PRISM' in clm_usrdat_name: + streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) + + bias_correct = nmlgen.get_value("bias_correct") + if bias_correct is not None: + streamlist.append(bias_correct) + anomaly_forcing = nmlgen.get_value("anomaly_forcing") + if anomaly_forcing[0] is not None: + streamlist += anomaly_forcing + + # Generate datm.streams.xml + outfile = os.path.join(confdir, "datm.streams"+inst_string+".xml" ) + schema_file = os.path.join(_CDEPS_CONFIG,"stream_definition_v2.0.xsd") + stream_file = os.path.join(_CDEPS_CONFIG,os.pardir, "datm","cime_config","stream_definition_datm.xml") + streams = StreamCDEPS(stream_file, schema_file) + streams.create_stream_xml(streamlist, case, outfile, data_list_path, + os.path.join(caseroot,'user_nl_datm_streams'+inst_string), + available_neon_data=available_neon_data) + + +#################################################################################### +def _create_drv_flds_in(case, confdir): +#################################################################################### + datm_preso3 = case.get_value("DATM_PRESO3") + + # for now we are hard-coding this file name and values because we only need it for ozone + if datm_preso3 != "none": + + # Generate drv_flds_in file + outfile = os.path.join(confdir, "drv_flds_in") + ozone_nl_name = "&ozone_coupling_nl" + ozone_freq_par = "atm_ozone_frequency" + ozone_freq_val = "'multiday_average'" + nl_fin = "/" + + with open(outfile, "w") as drv_fl: + drv_fl.write("{}\n".format(ozone_nl_name)) + drv_fl.write(" {} = {}\n".format(ozone_freq_par, ozone_freq_val)) + drv_fl.write("{}\n".format(nl_fin)) + + + +############################################################################### +def buildnml(case, caseroot, compname): +############################################################################### + rundir = case.get_value("RUNDIR") + inst_name = compname.upper()[1:] + ninst = case.get_value("NINST_"+inst_name) + if ninst is None: + ninst = case.get_value("NINST") + + # Determine configuration directory + confdir = os.path.join(caseroot,"Buildconf",compname + "conf") + if not os.path.isdir(confdir): + os.makedirs(confdir) + + #---------------------------------------------------- + # Construct the namelist generator + #---------------------------------------------------- + # Determine directory for user modified namelist_definitions.xml and namelist_defaults.xml + user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname) + expect (os.path.isdir(user_xml_dir), + "user_xml_dir {} does not exist ".format(user_xml_dir)) + + # NOTE: User definition *replaces* existing definition. + files = Files(comp_interface="nuopc") + definition_file = [files.get_value("NAMELIST_DEFINITION_FILE", {"component":compname})] + + user_definition = os.path.join(user_xml_dir, "namelist_definition_"+compname+".xml") + if os.path.isfile(user_definition): + definition_file = [user_definition] + for file_ in definition_file: + expect(os.path.isfile(file_), "Namelist XML file {} not found!".format(file_)) + + # Create the namelist generator object - independent of instance + nmlgen = NamelistGenerator(case, definition_file, files=files) + + #---------------------------------------------------- + # Clear out old data. + #---------------------------------------------------- + data_list_path = os.path.join(caseroot, "Buildconf", compname+".input_data_list") + if os.path.exists(data_list_path): + os.remove(data_list_path) + + #---------------------------------------------------- + # Loop over instances + #---------------------------------------------------- + for inst_counter in range(1, ninst+1): + # determine instance string + inst_string = "" + if ninst > 1: + inst_string = '_' + '{:04d}'.format(inst_counter) + + # If multi-instance case does not have restart file, use + # single-case restart for each instance + rpointer = "rpointer." + compname + if (os.path.isfile(os.path.join(rundir,rpointer)) and + (not os.path.isfile(os.path.join(rundir,rpointer + inst_string)))): + safe_copy(os.path.join(rundir, rpointer), + os.path.join(rundir, rpointer + inst_string)) + + inst_string_label = inst_string + if not inst_string_label: + inst_string_label = "\"\"" + + # create namelist output infile using user_nl_file as input + user_nl_file = os.path.join(caseroot, "user_nl_" + compname + inst_string) + expect(os.path.isfile(user_nl_file), + "Missing required user_nl_file {} ".format(user_nl_file)) + infile = os.path.join(confdir, "namelist_infile") + create_namelist_infile(case, user_nl_file, infile) + namelist_infile = [infile] + + # create namelist and stream file(s) data component + _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen, data_list_path) + + _create_drv_flds_in(case, confdir) + + # copy namelist files and stream text files, to rundir + copy_inputs_to_rundir(caseroot, compname, confdir, rundir, inst_string) + +############################################################################### +def get_user_nl_list(case): +############################################################################### + """Returns a list of user_nl_datm* files needed in this case + This function is called by CIME to stage the user_nl_datm* files in the case + directory. + """ + user_nl_list = ["user_nl_datm", "user_nl_datm_streams"] + return user_nl_list + +############################################################################### +def _main_func(): + caseroot = parse_input(sys.argv) + with Case(caseroot) as case: + buildnml(case, caseroot, "datm") + + +if __name__ == "__main__": + _main_func() From 60a32b716c16ad566e306c9cd22af9c4e04b6e70 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Mon, 27 Mar 2023 16:22:30 -0600 Subject: [PATCH 05/18] fix buildnml --- cime_config/buildnml | 330 -------------------------------------- datm/cime_config/buildnml | 5 +- 2 files changed, 1 insertion(+), 334 deletions(-) delete mode 100755 cime_config/buildnml diff --git a/cime_config/buildnml b/cime_config/buildnml deleted file mode 100755 index 2e915897..00000000 --- a/cime_config/buildnml +++ /dev/null @@ -1,330 +0,0 @@ -#!/usr/bin/env python3 - -"""Namelist creator for CDEPS data atm model. -""" - -# Typically ignore this. -# pylint: disable=invalid-name - -# Disable these because this is our standard setup -# pylint: disable=wildcard-import,unused-wildcard-import,wrong-import-position - -import os, sys - -_CDEPS_CONFIG = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir,os.pardir,"cime_config") -_CIMEROOT = os.environ.get("CIMEROOT") -if _CIMEROOT is None: - raise SystemExit("ERROR: must set CIMEROOT environment variable") -_LIBDIR = os.path.join(_CIMEROOT, "CIME", "Tools") -sys.path.append(_LIBDIR) -sys.path.append(_CDEPS_CONFIG) - -from datetime import datetime -from standard_script_setup import * -from CIME.case import Case -from CIME.nmlgen import NamelistGenerator -from CIME.utils import expect, safe_copy, SharedArea -from CIME.buildnml import create_namelist_infile, parse_input, copy_inputs_to_rundir -from CIME.XML.files import Files -from CIME.XML.inputdata import Inputdata -from stream_cdeps import StreamCDEPS - -logger = logging.getLogger(__name__) - -def _get_neon_data_availability(case, neonsite): - """ - Read the NEON inventory file and check availablity of tower data for this case. - Put available data in datavaliddate list and return it - Currently the valid date information is not used - """ - rundir = case.get_value("RUNDIR") - fullpath = os.path.join(rundir,"listing.csv") - if not os.path.exists(fullpath): - inputdata = Inputdata() - protocol, address, user, passwd, _, _, inventory = inputdata.get_next_server(attributes={'CLM_USRDAT_NAME':'NEON'}) - # CLM_USRDAT_NAME could also be NEON.PRISM - success = case.check_input_data(protocol=protocol, address=address, download=True) - logger.debug("_get_neon_data_availablity: address {} inventory is {} success {}".format(address, inventory, success)) - - server = CIME.Servers.WGET.wget_login(address, user, passwd) - if server: - with SharedArea(): - success = server.getfile(inventory, fullpath) - expect(success, "Could not download NEON data listing file from server") - - expect(os.path.exists(fullpath), "Could not download NEON data listing file from server") - datavaliddate = [] - oldestdate = None - newestdate = None - neonatm = None - dataversion = case.get_value("NEONVERSION") - if dataversion == "latest": - dataversions = ["v2", "v1"] - else: - dataversions = [dataversion] - - for version in dataversions: - if not newestdate: - with open(fullpath, "r") as fd: - for line in fd.readlines(): - fpath, fname = os.path.split(line) - if fpath.endswith(f'cdeps/{version}/{neonsite}'): - neonatm, date, _ = line.strip().split(',') - if neonatm.endswith('.nc'): - neonatm = neonatm.split('/')[-1] - newestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc") - if not oldestdate: - oldestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc") - neonatm = f'cdeps/{version}/{neonsite}/'+neonatm - datavaliddate.append(neonatm) - if newestdate: - logger.info("Found tower data version {} for {} through {}".format(version, oldestdate, newestdate)) - datavaliddate.sort() - return datavaliddate - - expect(newestdate, "No tower data found on server for NEON site {}".format(neonsite)) - return None - -# pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements -#################################################################################### -def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path): -#################################################################################### - """Write out the namelist for this component. - - Most arguments are the same as those for `NamelistGenerator`. The - `inst_string` argument is used as a suffix to distinguish files for - different instances. The `confdir` argument is used to specify the directory - in which output files will be placed. - """ - - #---------------------------------------------------- - # Write out datm_in and datm.streams.xml - #---------------------------------------------------- - - caseroot = case.get_value("CASEROOT") - datm_mode = case.get_value("DATM_MODE") - datm_topo = case.get_value("DATM_TOPO") - datm_presaero = case.get_value("DATM_PRESAERO") - datm_presndep = case.get_value("DATM_PRESNDEP") - datm_preso3 = case.get_value("DATM_PRESO3") - clm_usrdat_name = case.get_value("CLM_USRDAT_NAME") - datm_co2_tseries = case.get_value("DATM_CO2_TSERIES") - atm_grid = case.get_value("ATM_GRID") - model_grid = case.get_value("GRID") - comp_lnd = case.get_value("COMP_LND") - - # Check for incompatible options. - if "CLM" in datm_mode and comp_lnd == "clm": - expect(datm_presaero != "none", - "A DATM_MODE for CLM is incompatible with DATM_PRESAERO=none.") - expect(datm_presndep != "none", - "A DATM_MODE for CLM is incompatible with DATM_PRESNDEP=none.") - expect(datm_preso3 != "none", - "A DATM_MODE for CLM is incompatible with DATM_PRESO3=none.") - expect(datm_topo != "none", - "A DATM_MODE for CLM is incompatible with DATM_TOPO=none.") - - # Log some settings. - logger.debug("DATM mode is {}".format(datm_mode)) - logger.debug("DATM grid is {}".format(atm_grid)) - logger.debug("DATM presaero mode is {}".format(datm_presaero)) - logger.debug("DATM presndep mode is {}".format(datm_presndep)) - logger.debug("DATM preso3 mode is {}".format(datm_preso3)) - logger.debug("DATM topo mode is {}".format(datm_topo)) - logger.debug("CLM_USRDAT_NAME is {}".format(clm_usrdat_name)) - - # Initialize namelist defaults - config = {} - available_neon_data = None - if 'CLM_USRDAT' in model_grid: - config['model_grid'] = 'CLM_USRDAT' - neonsite = case.get_value("NEONSITE") - if neonsite: - config['neon'] = "True" - # download and use the listing.csv file to determine data availablity - available_neon_data = _get_neon_data_availability(case, neonsite) - else: - config['model_grid'] = model_grid - - config['datm_mode'] = datm_mode - config['datm_co2_tseries'] = datm_co2_tseries - config['datm_presaero'] = datm_presaero - config['datm_presndep'] = datm_presndep - config['datm_preso3'] = datm_preso3 - config['clm_usrdat_name'] = clm_usrdat_name - - if case.get_value('PTS_LON'): - scol_lon = float(case.get_value('PTS_LON')) - else: - scol_lon = -999. - if case.get_value('PTS_LAT'): - scol_lat = float(case.get_value('PTS_LAT')) - else: - scol_lat = -999. - if case.get_value('ATM_DOMAIN_FILE'): - if scol_lon > -999. and scol_lat > -999. and case.get_value("ATM_DOMAIN_FILE") != "UNSET": - config['single_column'] = 'true' - else: - config['single_column'] = 'false' - - nmlgen.init_defaults(infile, config) - - # Generate datm_in - namelist_file = os.path.join(confdir, "datm_in") - nmlgen.write_output_file(namelist_file, data_list_path, groups=['datm_nml']) - - # Determine streams - streamlist = nmlgen.get_streams() - if type(streamlist) == type(str()): - streamlist = [] - - if datm_presaero != "none": - streamlist.append("presaero.{}".format(datm_presaero)) - if datm_presndep != "none": - streamlist.append("presndep.{}".format(datm_presndep)) - if datm_preso3 != "none": - streamlist.append("preso3.{}".format(datm_preso3)) - if datm_topo != "none": - streamlist.append("topo.{}".format(datm_topo)) - if datm_co2_tseries != "none": - streamlist.append("co2tseries.{}".format(datm_co2_tseries)) - if 'PRISM' in clm_usrdat_name: - streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) - - bias_correct = nmlgen.get_value("bias_correct") - if bias_correct is not None: - streamlist.append(bias_correct) - anomaly_forcing = nmlgen.get_value("anomaly_forcing") - if anomaly_forcing[0] is not None: - streamlist += anomaly_forcing - - # Generate datm.streams.xml - outfile = os.path.join(confdir, "datm.streams"+inst_string+".xml" ) - schema_file = os.path.join(_CDEPS_CONFIG,"stream_definition_v2.0.xsd") - stream_file = os.path.join(_CDEPS_CONFIG,os.pardir, "datm","cime_config","stream_definition_datm.xml") - streams = StreamCDEPS(stream_file, schema_file) - streams.create_stream_xml(streamlist, case, outfile, data_list_path, - os.path.join(caseroot,'user_nl_datm_streams'+inst_string), - available_neon_data=available_neon_data) - - -#################################################################################### -def _create_drv_flds_in(case, confdir): -#################################################################################### - datm_preso3 = case.get_value("DATM_PRESO3") - - # for now we are hard-coding this file name and values because we only need it for ozone - if datm_preso3 != "none": - - # Generate drv_flds_in file - outfile = os.path.join(confdir, "drv_flds_in") - ozone_nl_name = "&ozone_coupling_nl" - ozone_freq_par = "atm_ozone_frequency" - ozone_freq_val = "'multiday_average'" - nl_fin = "/" - - with open(outfile, "w") as drv_fl: - drv_fl.write("{}\n".format(ozone_nl_name)) - drv_fl.write(" {} = {}\n".format(ozone_freq_par, ozone_freq_val)) - drv_fl.write("{}\n".format(nl_fin)) - - - -############################################################################### -def buildnml(case, caseroot, compname): -############################################################################### - rundir = case.get_value("RUNDIR") - inst_name = compname.upper()[1:] - ninst = case.get_value("NINST_"+inst_name) - if ninst is None: - ninst = case.get_value("NINST") - - # Determine configuration directory - confdir = os.path.join(caseroot,"Buildconf",compname + "conf") - if not os.path.isdir(confdir): - os.makedirs(confdir) - - #---------------------------------------------------- - # Construct the namelist generator - #---------------------------------------------------- - # Determine directory for user modified namelist_definitions.xml and namelist_defaults.xml - user_xml_dir = os.path.join(caseroot, "SourceMods", "src." + compname) - expect (os.path.isdir(user_xml_dir), - "user_xml_dir {} does not exist ".format(user_xml_dir)) - - # NOTE: User definition *replaces* existing definition. - files = Files(comp_interface="nuopc") - definition_file = [files.get_value("NAMELIST_DEFINITION_FILE", {"component":compname})] - - user_definition = os.path.join(user_xml_dir, "namelist_definition_"+compname+".xml") - if os.path.isfile(user_definition): - definition_file = [user_definition] - for file_ in definition_file: - expect(os.path.isfile(file_), "Namelist XML file {} not found!".format(file_)) - - # Create the namelist generator object - independent of instance - nmlgen = NamelistGenerator(case, definition_file, files=files) - - #---------------------------------------------------- - # Clear out old data. - #---------------------------------------------------- - data_list_path = os.path.join(caseroot, "Buildconf", compname+".input_data_list") - if os.path.exists(data_list_path): - os.remove(data_list_path) - - #---------------------------------------------------- - # Loop over instances - #---------------------------------------------------- - for inst_counter in range(1, ninst+1): - # determine instance string - inst_string = "" - if ninst > 1: - inst_string = '_' + '{:04d}'.format(inst_counter) - - # If multi-instance case does not have restart file, use - # single-case restart for each instance - rpointer = "rpointer." + compname - if (os.path.isfile(os.path.join(rundir,rpointer)) and - (not os.path.isfile(os.path.join(rundir,rpointer + inst_string)))): - safe_copy(os.path.join(rundir, rpointer), - os.path.join(rundir, rpointer + inst_string)) - - inst_string_label = inst_string - if not inst_string_label: - inst_string_label = "\"\"" - - # create namelist output infile using user_nl_file as input - user_nl_file = os.path.join(caseroot, "user_nl_" + compname + inst_string) - expect(os.path.isfile(user_nl_file), - "Missing required user_nl_file {} ".format(user_nl_file)) - infile = os.path.join(confdir, "namelist_infile") - create_namelist_infile(case, user_nl_file, infile) - namelist_infile = [infile] - - # create namelist and stream file(s) data component - _create_namelists(case, confdir, inst_string, namelist_infile, nmlgen, data_list_path) - - _create_drv_flds_in(case, confdir) - - # copy namelist files and stream text files, to rundir - copy_inputs_to_rundir(caseroot, compname, confdir, rundir, inst_string) - -############################################################################### -def get_user_nl_list(case): -############################################################################### - """Returns a list of user_nl_datm* files needed in this case - This function is called by CIME to stage the user_nl_datm* files in the case - directory. - """ - user_nl_list = ["user_nl_datm", "user_nl_datm_streams"] - return user_nl_list - -############################################################################### -def _main_func(): - caseroot = parse_input(sys.argv) - with Case(caseroot) as case: - buildnml(case, caseroot, "datm") - - -if __name__ == "__main__": - _main_func() diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index de9afac7..bcfb5b2e 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -189,10 +189,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path if datm_co2_tseries != "none": streamlist.append("co2tseries.{}".format(datm_co2_tseries)) if 'NEON' in clm_usrdat_name: - # streamlist.append("NEON.PRISM_PRECIP.CPER") - # streamlist.append("NEON.PRISM_PRECIP.".format(clm_usrdat_name[-4:])) # this ends up being .RISM because the last for of NEON.PRISM are RISM - # streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) # USE THIS VERSION ONCE IT DOESN'T CAUSE SVN FAILURES - streamlist.append("NEON.PRISM_PRECIP."+neonsite) + streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) bias_correct = nmlgen.get_value("bias_correct") if bias_correct is not None: From 99f4dc3e7c31c553141a8bc5556d5fa830dabaa7 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Wed, 29 Mar 2023 15:13:09 -0600 Subject: [PATCH 06/18] include third datm stream for NEON --- cime_config/stream_cdeps.py | 12 +++--- datm/cime_config/buildnml | 11 ++++- datm/cime_config/namelist_definition_datm.xml | 2 +- datm/cime_config/stream_definition_datm.xml | 41 +++++++++++++++++-- 4 files changed, 54 insertions(+), 12 deletions(-) diff --git a/cime_config/stream_cdeps.py b/cime_config/stream_cdeps.py index f32b294a..144df954 100644 --- a/cime_config/stream_cdeps.py +++ b/cime_config/stream_cdeps.py @@ -135,17 +135,19 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file # write contents of stream file for stream_name in stream_names: - if stream_name.startswith("NEON.PRISM"): + # include NEON.$NEONSITE non-precipitation data streams whether use PRISM or NEON precip + if stream_name.startswith("NEON.") and ('PRECIP' not in stream_name): + self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.$NEONSITE"}, + err_msg="No stream_entry {} found".format(stream_name)) + elif stream_name.startswith("NEON.PRISM_PRECIP"): self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.PRISM_PRECIP.$NEONSITE"}, err_msg="No stream_entry {} found".format(stream_name)) - elif stream_name.startswith("NEON."): - self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.$NEONSITE"}, + elif stream_name.startswith("NEON.NEON_PRECIP"): + self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.NEON_PRECIP.$NEONSITE"}, err_msg="No stream_entry {} found".format(stream_name)) - elif stream_name.startswith("CLM_USRDAT."): self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "CLM_USRDAT.$CLM_USRDAT_NAME"}, err_msg="No stream_entry {} found".format(stream_name)) - elif stream_name: self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : stream_name}, err_msg="No stream_entry {} found".format(stream_name)) diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index bcfb5b2e..6b7664cb 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -177,7 +177,7 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path streamlist = nmlgen.get_streams() if type(streamlist) == type(str()): streamlist = [] - + logger.info("streamlist prior to additions:{}".format(streamlist)) if datm_presaero != "none": streamlist.append("presaero.{}".format(datm_presaero)) if datm_presndep != "none": @@ -188,8 +188,15 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path streamlist.append("topo.{}".format(datm_topo)) if datm_co2_tseries != "none": streamlist.append("co2tseries.{}".format(datm_co2_tseries)) - if 'NEON' in clm_usrdat_name: + if clm_usrdat_name == 'NEON.PRISM': streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) + logger.info("appended {} to streamlist".format(clm_usrdat_name+"_PRECIP."+neonsite)) + if clm_usrdat_name == 'NEON': + streamlist.append(clm_usrdat_name+".NEON_PRECIP."+neonsite) + #if 'NEON' in clm_usrdat_name: + # streamlist.append("NEON."+neonsite) + # non-precip datm stream should already be in usermods for any neon case + logger.info('full streamlist: {}'.format(streamlist)) bias_correct = nmlgen.get_value("bias_correct") if bias_correct is not None: diff --git a/datm/cime_config/namelist_definition_datm.xml b/datm/cime_config/namelist_definition_datm.xml index d59156e1..c677bffe 100644 --- a/datm/cime_config/namelist_definition_datm.xml +++ b/datm/cime_config/namelist_definition_datm.xml @@ -38,7 +38,7 @@ 1PT.urbanc_alpha - NEON.$NEONSITE,NEON.PRISM_PRECIP.$NEONSITE + NEON.$NEONSITE CLM_USRDAT.$CLM_USRDAT_NAME diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 1e95074c..20c6f8e0 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -236,7 +236,6 @@ $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc - FSDS Faxa_swdn ZBOT Sa_z TBOT Sa_tbot @@ -267,7 +266,6 @@ single - @@ -304,6 +302,41 @@ single + + + + + + + none + + + $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc + + + PRECTmms Faxa_precn + + null + + none + + null + $DATM_YR_ALIGN + $DATM_YR_START + $DATM_YR_END + 0 + + linear + + + cycle + limit + + + 1.5 + + single + @@ -816,12 +849,12 @@ - + none - $DIN_LOC_ROOT_CLMFORC/$CLM_USRDAT_NAME/CLM1PT_data/%ym.nc + $DIN_LOC_ROOT_CLMFORC/NEON/CLM1PT_data/%ym.nc PRECTmms Faxa_precn From 0f087f578a3d01c66bba8567245b23c05540ae90 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 30 Mar 2023 16:12:06 -0600 Subject: [PATCH 07/18] datafile fix for PRISM_PRECIP --- cime_config/stream_cdeps.py | 2 +- datm/cime_config/stream_definition_datm.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cime_config/stream_cdeps.py b/cime_config/stream_cdeps.py index 144df954..8bf77525 100644 --- a/cime_config/stream_cdeps.py +++ b/cime_config/stream_cdeps.py @@ -181,7 +181,7 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file stream_vars[node_name] = "" stream_datafiles = "" for child in self.get_children(root=node): - if available_neon_data and stream_name.startswith("NEON"): + if available_neon_data and stream_name.startswith("NEON") and ('PRISM' not in stream_name): rundir = case.get_value("RUNDIR") for neon in available_neon_data: stream_datafiles += os.path.join(rundir,"inputdata","atm",neon) + "\n" diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 20c6f8e0..d0406396 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -275,7 +275,7 @@ none - /glade/u/home/tking/neon/prism_$NEONSITE_%y.nc + /glade/u/home/tking/neon/prism_${NEONSITE}_%y.nc PRECIP Faxa_precn From e6676be625d732041eb145e6a7f235e8bc15d1e4 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 6 Apr 2023 11:25:37 -0600 Subject: [PATCH 08/18] add tests, use in stream nl --- cime_config/testdefs/testlist_cdeps.xml | 10 +++++++++- datm/cime_config/buildnml | 3 --- datm/cime_config/stream_definition_datm.xml | 8 ++++---- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cime_config/testdefs/testlist_cdeps.xml b/cime_config/testdefs/testlist_cdeps.xml index 03cbfb0c..93d2526b 100644 --- a/cime_config/testdefs/testlist_cdeps.xml +++ b/cime_config/testdefs/testlist_cdeps.xml @@ -88,7 +88,7 @@ - + @@ -97,6 +97,14 @@ + + + + + + + + diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index 6b7664cb..94be79dc 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -193,9 +193,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path logger.info("appended {} to streamlist".format(clm_usrdat_name+"_PRECIP."+neonsite)) if clm_usrdat_name == 'NEON': streamlist.append(clm_usrdat_name+".NEON_PRECIP."+neonsite) - #if 'NEON' in clm_usrdat_name: - # streamlist.append("NEON."+neonsite) - # non-precip datm stream should already be in usermods for any neon case logger.info('full streamlist: {}'.format(streamlist)) bias_correct = nmlgen.get_value("bias_correct") diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index d0406396..44a27d26 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -275,7 +275,7 @@ none - /glade/u/home/tking/neon/prism_${NEONSITE}_%y.nc + /glade/u/home/tking/neon/prism_${NEONSITE}_%y.nc PRECIP Faxa_precn @@ -311,7 +311,7 @@ none - $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc + $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc PRECTmms Faxa_precn @@ -849,12 +849,12 @@ - + none - $DIN_LOC_ROOT_CLMFORC/NEON/CLM1PT_data/%ym.nc + $DIN_LOC_ROOT_CLMFORC/$CLM_USRDAT_NAME/CLM1PT_data/%ym.nc PRECTmms Faxa_precn From a6ec34f2be6fdd61ddd3f07ba7d5bea3d6bada10 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 6 Apr 2023 11:27:44 -0600 Subject: [PATCH 09/18] minor bug --- datm/cime_config/stream_definition_datm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 44a27d26..7e944412 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -849,7 +849,7 @@ - + none From 53b75f0424ce90ae8080f043b47218ed31cfd4dd Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 7 Apr 2023 13:01:05 -0600 Subject: [PATCH 10/18] cleanup logger info --- cime_config/testdefs/testlist_cdeps.xml | 2 +- datm/cime_config/buildnml | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/cime_config/testdefs/testlist_cdeps.xml b/cime_config/testdefs/testlist_cdeps.xml index 93d2526b..1b4a8d48 100644 --- a/cime_config/testdefs/testlist_cdeps.xml +++ b/cime_config/testdefs/testlist_cdeps.xml @@ -88,7 +88,7 @@ - + diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index 94be79dc..4ac3f156 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -42,7 +42,6 @@ def _get_neon_data_availability(case, neonsite): if not os.path.exists(fullpath): inputdata = Inputdata() protocol, address, user, passwd, _, _, inventory = inputdata.get_next_server(attributes={'CLM_USRDAT_NAME':'NEON'}) - # CLM_USRDAT_NAME could also be NEON.PRISM success = case.check_input_data(protocol=protocol, address=address, download=True) logger.debug("_get_neon_data_availablity: address {} inventory is {} success {}".format(address, inventory, success)) @@ -177,7 +176,6 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path streamlist = nmlgen.get_streams() if type(streamlist) == type(str()): streamlist = [] - logger.info("streamlist prior to additions:{}".format(streamlist)) if datm_presaero != "none": streamlist.append("presaero.{}".format(datm_presaero)) if datm_presndep != "none": @@ -190,10 +188,8 @@ def _create_namelists(case, confdir, inst_string, infile, nmlgen, data_list_path streamlist.append("co2tseries.{}".format(datm_co2_tseries)) if clm_usrdat_name == 'NEON.PRISM': streamlist.append(clm_usrdat_name+"_PRECIP."+neonsite) - logger.info("appended {} to streamlist".format(clm_usrdat_name+"_PRECIP."+neonsite)) if clm_usrdat_name == 'NEON': streamlist.append(clm_usrdat_name+".NEON_PRECIP."+neonsite) - logger.info('full streamlist: {}'.format(streamlist)) bias_correct = nmlgen.get_value("bias_correct") if bias_correct is not None: From 0cbf0abc3939e5fcac39f04d84132bbc79cb78bc Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 13 Apr 2023 08:12:20 -0600 Subject: [PATCH 11/18] inputdata stream adjustment in xml; CDEPS testdefs/shell commands --- datm/cime_config/stream_definition_datm.xml | 2 +- datm/cime_config/testdefs/testlist_datm.xml | 8 ++++++++ .../testdefs/testmods_dirs/datm/PRISM/shell_commands | 1 + 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 7e944412..6696f7ee 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -275,7 +275,7 @@ none - /glade/u/home/tking/neon/prism_${NEONSITE}_%y.nc + $DIN_LOC_ROOT/atm/cdeps/v2/$NEONSITE/prism_%y.nc PRECIP Faxa_precn diff --git a/datm/cime_config/testdefs/testlist_datm.xml b/datm/cime_config/testdefs/testlist_datm.xml index ce9efb45..e16ac210 100644 --- a/datm/cime_config/testdefs/testlist_datm.xml +++ b/datm/cime_config/testdefs/testlist_datm.xml @@ -102,6 +102,14 @@ + + + + + + + + diff --git a/datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands b/datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands new file mode 100644 index 00000000..13bbc00d --- /dev/null +++ b/datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands @@ -0,0 +1 @@ +./xmlchange CLM_USRDAT_NAME=NEON.PRISM From d97ebc3f1c4141ad54ec10570d66f6b4d8d3fbbe Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 13 Apr 2023 08:14:03 -0600 Subject: [PATCH 12/18] remove prism test without shell command --- cime_config/testdefs/testlist_cdeps.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/cime_config/testdefs/testlist_cdeps.xml b/cime_config/testdefs/testlist_cdeps.xml index 1b4a8d48..03cbfb0c 100644 --- a/cime_config/testdefs/testlist_cdeps.xml +++ b/cime_config/testdefs/testlist_cdeps.xml @@ -97,14 +97,6 @@ - - - - - - - - From f6f4c22e47f978813a6903ee703cc312e3f22eb9 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 13 Apr 2023 17:01:21 -0600 Subject: [PATCH 13/18] inputdata updates --- datm/cime_config/stream_definition_datm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 6696f7ee..4e85d520 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -275,7 +275,7 @@ none - $DIN_LOC_ROOT/atm/cdeps/v2/$NEONSITE/prism_%y.nc + $DIN_LOC_ROOT/atm/cdeps/PRISM/{$NEONSITE}_%y.nc PRECIP Faxa_precn From 02f3e3a06c45cf4317532ea08df59add04ffd22d Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 14 Apr 2023 13:47:12 -0600 Subject: [PATCH 14/18] update to include v3 --- datm/cime_config/buildnml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index 4ac3f156..3a971512 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -58,7 +58,7 @@ def _get_neon_data_availability(case, neonsite): neonatm = None dataversion = case.get_value("NEONVERSION") if dataversion == "latest": - dataversions = ["v2", "v1"] + dataversions = ["v3", "v2", "v1"] else: dataversions = [dataversion] From 36b9b1948ac178e9e897be3891450b979549179a Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 18 Apr 2023 12:07:46 -0600 Subject: [PATCH 15/18] remove cdeps tests --- datm/cime_config/testdefs/testlist_datm.xml | 8 -------- .../testdefs/testmods_dirs/datm/PRISM/shell_commands | 1 - 2 files changed, 9 deletions(-) delete mode 100644 datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands diff --git a/datm/cime_config/testdefs/testlist_datm.xml b/datm/cime_config/testdefs/testlist_datm.xml index e16ac210..ce9efb45 100644 --- a/datm/cime_config/testdefs/testlist_datm.xml +++ b/datm/cime_config/testdefs/testlist_datm.xml @@ -102,14 +102,6 @@ - - - - - - - - diff --git a/datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands b/datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands deleted file mode 100644 index 13bbc00d..00000000 --- a/datm/cime_config/testdefs/testmods_dirs/datm/PRISM/shell_commands +++ /dev/null @@ -1 +0,0 @@ -./xmlchange CLM_USRDAT_NAME=NEON.PRISM From 1a71713b0d1e3a3906bd160a471dae654e901642 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Thu, 20 Apr 2023 14:22:10 -0600 Subject: [PATCH 16/18] include comment v1 --- datm/cime_config/stream_definition_datm.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index 4e85d520..e832f269 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -233,6 +233,7 @@ none + $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc @@ -311,6 +312,7 @@ none + $DIN_LOC_ROOT/atm/cdeps/v1/$NEONSITE/%ym.nc From 088436d04f0df971f3e98573c2fbff76c5488fb6 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 26 Apr 2023 09:53:56 -0600 Subject: [PATCH 17/18] Syntax error for the prism data --- datm/cime_config/stream_definition_datm.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datm/cime_config/stream_definition_datm.xml b/datm/cime_config/stream_definition_datm.xml index e832f269..4da427f2 100644 --- a/datm/cime_config/stream_definition_datm.xml +++ b/datm/cime_config/stream_definition_datm.xml @@ -276,7 +276,7 @@ none - $DIN_LOC_ROOT/atm/cdeps/PRISM/{$NEONSITE}_%y.nc + $DIN_LOC_ROOT/atm/cdeps/PRISM/${NEONSITE}_%y.nc PRECIP Faxa_precn From cd96aace91c101d6d36b50c102e81182143dca0f Mon Sep 17 00:00:00 2001 From: Teagan King <98482480+TeaganKing@users.noreply.github.com> Date: Wed, 26 Apr 2023 10:19:01 -0600 Subject: [PATCH 18/18] buildnml prism version loop comment --- datm/cime_config/buildnml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/datm/cime_config/buildnml b/datm/cime_config/buildnml index 3a971512..c3d59375 100755 --- a/datm/cime_config/buildnml +++ b/datm/cime_config/buildnml @@ -63,7 +63,11 @@ def _get_neon_data_availability(case, neonsite): dataversions = [dataversion] for version in dataversions: - if not newestdate: + # Once a date range for a version has been determined for any version, the loop is complete. + # Eg, if data is available for v3, the loop will not continue to v2; + # however, if data is not available for v3, it will check for v2. + # Thus, it is important for dataversions to be listed from newest to oldest. + if not newestdate: with open(fullpath, "r") as fd: for line in fd.readlines(): fpath, fname = os.path.split(line)