diff --git a/.gitignore b/.gitignore index 5739c58..c7f1f7a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ *.pyc demo/MRI-JMA-JRA55-do-1-3/testFiles demo/MRI-JMA-JRA55-do-1-3/input4MIPs +demo/IIS-UT-GSWP3-LS3MIP-1-1/input4MIPs +demo/IIS-UT-GSWP3-LS3MIP-1-1/_obsolete \ No newline at end of file diff --git a/Tables/input4MIPs_A3hr.json b/Tables/input4MIPs_A3hr.json index 3ceeab5..545f0e2 100644 --- a/Tables/input4MIPs_A3hr.json +++ b/Tables/input4MIPs_A3hr.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_A3hr" }, "variable_entry":{ diff --git a/Tables/input4MIPs_A3hrPt.json b/Tables/input4MIPs_A3hrPt.json index c139e80..48d6b4f 100644 --- a/Tables/input4MIPs_A3hrPt.json +++ b/Tables/input4MIPs_A3hrPt.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_A3hrPt" }, "variable_entry":{ diff --git a/Tables/input4MIPs_Afx.json b/Tables/input4MIPs_Afx.json index 7b1bde0..85b1bd9 100644 --- a/Tables/input4MIPs_Afx.json +++ b/Tables/input4MIPs_Afx.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos land", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Afx" }, "variable_entry":{ diff --git a/Tables/input4MIPs_CV.json b/Tables/input4MIPs_CV.json index badcd26..6c58208 100644 --- a/Tables/input4MIPs_CV.json +++ b/Tables/input4MIPs_CV.json @@ -87,6 +87,7 @@ "CNRM-Cerfacs":"CNRM (Centre National de Recherches Meteorologiques, Toulouse 31057, France), CERFACS (Centre Europeen de Recherche et de Formation Avancee en Calcul Scientifique, Toulouse 31100, France)", "IACETH":"Institute for Atmosphere and Climate, ETH Zurich, Zurich 8092, Switzerland", "IAMC":"Integrated Assessment Modeling Consortium (see www.globalchange.umd.edu/iamc/membership for complete membership). Mailing address: International Institute for Applied Systems Analysis (IIASA), Schlossplatz 1, A-2361 Laxenburg, Austria", + "IIS-UTokyo":"Institute of Industrial Science, The University of Tokyo, Hongo, Bunkyo-ku 113-8656, Japan", "ImperialCollege":"Imperial College London, South Kensington Campus, London SW7 2AZ, UK", "MOHC":"Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK", "MPI-B":"Max Planck Institute for Biogeochemistry, Jena 07745, Germany", @@ -240,6 +241,38 @@ "variable_id" ], "source_id":{ + "IIS-UTokyo-GSWP3-LS3MIP-1-1":{ + "comment":"Atmospheric boundary condition for retrospective off-line experiment of Global Soil Wetness Project Phase 3 (GSWP3, 1901-01 to 2014-12)", + "contact":"Hyungjun Kim (hjkim@iis.u-tokyo.ac.jp)", + "dataset_category":"atmosphericState", + "further_info_url":"https://hyungjun.github.io/GSWP3.DataDescription/; http://search.diasjp.net/en/dataset/GSWP3_EXP1_Forcing", + "grid":"0.5x0.5 degree latitude x longitude", + "grid_label":"gn", + "institution_id":"IIS-UTokyo", + "nominal_resolution":"50 km", + "product":"bias corrected & downscaled reanalysis", + "references":"Kim 2017: Global Soil Wetness Project Phase 3 Atmospheric Boundary Conditions (Experiment 1) [Data set]. Data Integration and Analysis System (DIAS). https://doi.org/10.20783/DIAS.501, ", + "region":[ + "global_land" + ], + "release_year":"2019", + "source":"GSWP3-EXP1 for LS3MIP-landhist (v1.1): Atmospheric boundary conditions for landhist experiment of LS3MIP", + "source_description":"Land state datasets produced by the University of Tokyo for the LS3MIP landhist experiment of CMIP6", + "source_id":"IIS-UTokyo-GSWP3-LS3MIP-1-1", + "source_variables":[ + "huss", + "prra", + "prsn", + "psl", + "rlds", + "rsds", + "sfcWind", + "tas" + ], + "source_version":"1.1", + "target_mip":"LS3MIP", + "title":"IIS-UTokyo GSWP3 LS3MIP 1.1 dataset prepared for input4MIPs" + }, "MRI-JRA55-do-1-3":{ "source":"MRI JRA55-do 1.3: Atmospheric state generated for OMIP based on the JRA-55 reanalysis" }, diff --git a/Tables/input4MIPs_LIday.json b/Tables/input4MIPs_LIday.json index 256b1bb..878128c 100644 --- a/Tables/input4MIPs_LIday.json +++ b/Tables/input4MIPs_LIday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIday" }, "variable_entry":{ diff --git a/Tables/input4MIPs_LIfx.json b/Tables/input4MIPs_LIfx.json index f70c19d..8db492f 100644 --- a/Tables/input4MIPs_LIfx.json +++ b/Tables/input4MIPs_LIfx.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIfx" }, "variable_entry":{ diff --git a/Tables/input4MIPs_LIyrC.json b/Tables/input4MIPs_LIyrC.json index af0fc5c..a67c965 100644 --- a/Tables/input4MIPs_LIyrC.json +++ b/Tables/input4MIPs_LIyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIyrC" }, "variable_entry":{ diff --git a/Tables/input4MIPs_Lday.json b/Tables/input4MIPs_Lday.json index 0524ab0..87464e6 100644 --- a/Tables/input4MIPs_Lday.json +++ b/Tables/input4MIPs_Lday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"land", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Lday" }, "variable_entry":{ diff --git a/Tables/input4MIPs_Oday.json b/Tables/input4MIPs_Oday.json index 6ef7425..da74a3a 100644 --- a/Tables/input4MIPs_Oday.json +++ b/Tables/input4MIPs_Oday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Oday" }, "variable_entry":{ diff --git a/Tables/input4MIPs_Ofx.json b/Tables/input4MIPs_Ofx.json index 48c0d91..9649eda 100644 --- a/Tables/input4MIPs_Ofx.json +++ b/Tables/input4MIPs_Ofx.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Ofx" }, "variable_entry":{ diff --git a/Tables/input4MIPs_Omon.json b/Tables/input4MIPs_Omon.json index df3fab2..1fe5a5a 100644 --- a/Tables/input4MIPs_Omon.json +++ b/Tables/input4MIPs_Omon.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Omon" }, "variable_entry":{ diff --git a/Tables/input4MIPs_OmonC.json b/Tables/input4MIPs_OmonC.json index eb4ccb5..dc232e0 100644 --- a/Tables/input4MIPs_OmonC.json +++ b/Tables/input4MIPs_OmonC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OmonC" }, "variable_entry":{ diff --git a/Tables/input4MIPs_OyrC.json b/Tables/input4MIPs_OyrC.json index 7e3e367..d5fa909 100644 --- a/Tables/input4MIPs_OyrC.json +++ b/Tables/input4MIPs_OyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OyrC" }, "variable_entry":{ diff --git a/Tables/input4MIPs_SI3hrPt.json b/Tables/input4MIPs_SI3hrPt.json index 114b394..64517f6 100644 --- a/Tables/input4MIPs_SI3hrPt.json +++ b/Tables/input4MIPs_SI3hrPt.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SI3hrPt" }, "variable_entry":{ diff --git a/Tables/input4MIPs_SIday.json b/Tables/input4MIPs_SIday.json index e5effad..eb89404 100644 --- a/Tables/input4MIPs_SIday.json +++ b/Tables/input4MIPs_SIday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SIday" }, "variable_entry":{ diff --git a/Tables/input4MIPs_SImon.json b/Tables/input4MIPs_SImon.json index 7402eaf..c4a6724 100644 --- a/Tables/input4MIPs_SImon.json +++ b/Tables/input4MIPs_SImon.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SImon" }, "variable_entry":{ diff --git a/Tables/input4MIPs_grids.json b/Tables/input4MIPs_grids.json index 86daf81..edf4aaa 100644 --- a/Tables/input4MIPs_grids.json +++ b/Tables/input4MIPs_grids.json @@ -5,7 +5,7 @@ "data_specs_version":"01.00.29", "missing_value":"1e20", "product":"input4MIPs", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_grids" }, "axis_entry":{ diff --git a/demo/IIS-UT-GSWP3-LS3MIP-1-1/Tables b/demo/IIS-UT-GSWP3-LS3MIP-1-1/Tables new file mode 120000 index 0000000..561a38f --- /dev/null +++ b/demo/IIS-UT-GSWP3-LS3MIP-1-1/Tables @@ -0,0 +1 @@ +../../Tables \ No newline at end of file diff --git a/demo/IIS-UT-GSWP3-LS3MIP-1-1/gswp3-input.json b/demo/IIS-UT-GSWP3-LS3MIP-1-1/gswp3-input.json new file mode 100644 index 0000000..1838e31 --- /dev/null +++ b/demo/IIS-UT-GSWP3-LS3MIP-1-1/gswp3-input.json @@ -0,0 +1,34 @@ +{ + "_control_vocabulary_file": "input4MIPs_CV.json", + "_AXIS_ENTRY_FILE": "input4MIPs_coordinate.json", + "_FORMULA_VAR_FILE": "input4MIPs_formula_terms.json", + "activity_id": "input4MIPs", + + "calendar": "gregorian", + "dataset_category": "atmosphericState", + "frequency": "3hr", + "grid": "0.5x0.5 degree latitude x longitude", + "grid_label": "gn", + "nominal_resolution": "50 km", + "product": "bias corrected & downscaled reanalysis", + "realm": "land", + "region": "global_land", + "target_mip": "LS3MIP", + + "comment": "Atmospheric boundary condition for retrospective off-line experiment of Global Soil Wetness Project Phase 3 (GSWP3, 1901-01 to 2014-12)", + "contact ": "Hyungjun Kim (hjkim@iis.u-tokyo.ac.jp)", + "further_info_url": "https://hyungjun.github.io/GSWP3.DataDescription/; http://search.diasjp.net/en/dataset/GSWP3_EXP1_Forcing", + "institution_id": "IIS-UTokyo", + "references": "Kim 2017: Global Soil Wetness Project Phase 3 Atmospheric Boundary Conditions (Experiment 1) [Data set]. Data Integration and Analysis System (DIAS). https://doi.org/10.20783/DIAS.501", + "source_id": "IIS-UTokyo-GSWP3-LS3MIP-1-1", + "source": "GSWP3-EXP1 for LS3MIP-landhis (v1.1): Atmospheric boundary conditions for landhist experiment of LS3MIP", + "source_version": "1.1", + "title": "IIS-UTokyo GSWP3-LS3MIP dataset prepared for input4MIPs", + + "outpath": ".", + "_history_template": "%s; CMOR rewrote data to be consistent with , CMIP6, and standards", + "output_path_template": "", + "output_file_template": "", + "tracking_prefix": "hdl:21.14100", + "license": "IIS-UTokyo-GSWP3-LS3MIP by H. Kim is licensed under a Creative Commons Attribution-[NonCommercial-]ShareAlike 4.0 International License (https://creativecommons.org/licenses). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing input4MIPs output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file). The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law." +} diff --git a/demo/IIS-UT-GSWP3-LS3MIP-1-1/runCmorDemo.py b/demo/IIS-UT-GSWP3-LS3MIP-1-1/runCmorDemo.py new file mode 100644 index 0000000..d8b3ff9 --- /dev/null +++ b/demo/IIS-UT-GSWP3-LS3MIP-1-1/runCmorDemo.py @@ -0,0 +1,229 @@ +import cmor,json,os +import cdms2 as cdm +#import cdtime as cdt +import numpy as np +#from calendar import isleap +#cdm.setAutoBounds('on') # Caution, this attempts to automatically set coordinate bounds - please check outputs using this option +#import pdb ; # Debug statement - import if enabling debugging below + +# Notes +# PJD 5 Feb 2018 - Started +# PJD 21 Feb 2018 - Updated for CMOR3.3.1 +# PJD 24 Feb 2018 - Updated paths for demo dir +# PJD 7 Mar 2018 - Updated pr to prra +# PJD 8 Mar 2018 - Correct prra 'positive'; Added ficeberg2d placeholder +# PJD 5 Apr 2018 - Update for latest variable list +# PJD 20 Jan 2019 - Updated for IIS-UTokyo data +# TODO: Fix missing_value assignment problem + +#%% Create input decks for all variables - 'fileList' will need to be amended to include all files 1958-2018 +inputDict = {} +# The following are temporal average measurements at 3hr intervals (not point values) +inputDict['A3hr'] = {} +key = 'Rainf' +inputDict['A3hr'][key] = {} +inputDict['A3hr'][key]['fileList'] = [ + 'GSWP3.BC.Rainf.3hrMap.1901.nc', # All these 'fileList' entries will require updating to include all files to be processed + ] +inputDict['A3hr'][key]['inputVarName'] = 'Rainf' +inputDict['A3hr'][key]['outputVarName'] = 'prra' ; # Was pr +inputDict['A3hr'][key]['outputUnits'] = 'kg m-2 s-1' +inputDict['A3hr'][key]['positive'] = None +key = 'Snowf' +inputDict['A3hr'][key] = {} +inputDict['A3hr'][key]['fileList'] = [ + 'GSWP3.BC.Snowf.3hrMap.1901.nc' + ] +inputDict['A3hr'][key]['inputVarName'] = 'Snowf' +inputDict['A3hr'][key]['outputVarName'] = 'prsn' +inputDict['A3hr'][key]['outputUnits'] = 'kg m-2 s-1' +inputDict['A3hr'][key]['positive'] = None +key = 'LWdown' +inputDict['A3hr'][key] = {} +inputDict['A3hr'][key]['fileList'] = [ + 'GSWP3.BC.LWdown.3hrMap.1901.nc' + ] +inputDict['A3hr'][key]['inputVarName'] = 'LWdown' +inputDict['A3hr'][key]['outputVarName'] = 'rlds' +inputDict['A3hr'][key]['outputUnits'] = 'W m-2' +inputDict['A3hr'][key]['positive'] = 'down' +key = 'SWdown' +inputDict['A3hr'][key] = {} +inputDict['A3hr'][key]['fileList'] = [ + 'GSWP3.BC.SWdown.3hrMap.1901.nc' + ] +inputDict['A3hr'][key]['inputVarName'] = 'SWdown' +inputDict['A3hr'][key]['outputVarName'] = 'rsds' +inputDict['A3hr'][key]['outputUnits'] = 'W m-2' +inputDict['A3hr'][key]['positive'] = 'down' +#key = 'SfcWind' +#inputDict['A3hr'][key] = {} +#inputDict['A3hr'][key]['fileList'] = [ +# 'GSWP3.BC.Wind.3hrMap.1901.nc' +# ] +#inputDict['A3hr'][key]['inputVarName'] = 'Wind' +#inputDict['A3hr'][key]['outputVarName'] = 'SfcWind' +#inputDict['A3hr'][key]['outputUnits'] = 'm s-1' +#inputDict['A3hr'][key]['positive'] = None +# The following are point measurements at 3hr intervals (not temporal averages) +inputDict['A3hrPt'] = {} +key = 'Qair' +inputDict['A3hrPt'][key] = {} +inputDict['A3hrPt'][key]['fileList'] = [ + 'GSWP3.BC.Qair.3hrMap.1901.nc' + ] +inputDict['A3hrPt'][key]['inputVarName'] = 'Qair' +inputDict['A3hrPt'][key]['outputVarName'] = 'huss' +inputDict['A3hrPt'][key]['outputUnits'] = '1.0' +inputDict['A3hrPt'][key]['positive'] = None +key = 'Psurf' +inputDict['A3hrPt'][key] = {} +inputDict['A3hrPt'][key]['fileList'] = [ + 'GSWP3.BC.PSurf.3hrMap.1901.nc' + ] +inputDict['A3hrPt'][key]['inputVarName'] = 'PSurf' +inputDict['A3hrPt'][key]['outputVarName'] = 'psl' +inputDict['A3hrPt'][key]['outputUnits'] = 'Pa' +inputDict['A3hrPt'][key]['positive'] = None +key = 'Tair' +inputDict['A3hrPt'][key] = {} +inputDict['A3hrPt'][key]['fileList'] = [ + 'GSWP3.BC.Tair.3hrMap.1901.nc' + ] +inputDict['A3hrPt'][key]['inputVarName'] = 'Tair' +inputDict['A3hrPt'][key]['outputVarName'] = 'tas' +inputDict['A3hrPt'][key]['outputUnits'] = 'K' +inputDict['A3hrPt'][key]['positive'] = None + +downloadPath = '/work/durack1/Shared/160427_CMIP6_Forcing/HyungjunKim_LS3MIP/190120' + +#%% Loop through entries and process file lists +for key in inputDict.keys(): + # User provided input + cmorTable = ''.join(['Tables/input4MIPs_',key,'.json']) ; # Aday,Amon,Lmon,Omon,SImon,fx - Load target table, axis info (coordinates, grid*) and CVs + cmorJson = json.load(open(cmorTable)) + inputJson = 'gswp3-input.json' ; # Update contents of this file to set your global_attributes + newJson = json.load(open(inputJson)) + for var in inputDict[key].keys(): + outVar = inputDict[key][var]['outputVarName'] + # Update frequency based on variableand write output to CMOR input file + newJson['frequency'] = cmorJson['variable_entry'][outVar]['frequency'] + json.dump(newJson,open('tmp.json','w'),ensure_ascii=True,encoding='utf-8',sort_keys=True) + inputVarName = inputDict[key][var]['inputVarName'] + outputVarName = inputDict[key][var]['outputVarName'] + outputUnits = inputDict[key][var]['outputUnits'] + for count,filePath in enumerate(inputDict[key][var]['fileList']): + print 'key,var:',key,var + print 'filePath:',filePath + print 'cmorTable:',cmorTable + print 'inputVarName:',inputVarName + print 'outputVarName:',outputVarName + print 'outputUnits:',outputUnits + # Open and read input netcdf file + f = cdm.open(os.path.join(downloadPath,filePath)) + print 'Source data read start..' + d = f(inputVarName) ; # Or use temporal subset for testing below + #d = f(inputVarName,time=slice(0,5)) + print 'Source data read end..' + # Reset missing value + d.setMissing(1e20) ; # This should also set fill_value, and suppress CMOR variable history being written + # Get axes + lat = d.getLatitude() + lon = d.getLongitude() + time = d.getTime() ; # Assumes variable is named 'time', for the demo file this is named 'months' + #time = d.getAxis(0) ; # Rather use a file dimension-based load + + # CMOR requires a time axis with bounds is passed, create this using cdt + time.setBounds(time.genGenericBounds()) + + #%% Initialize and run CMOR + print 'Start CMORizing..' + # For more information see https://cmor.llnl.gov/mydoc_cmor3_api/ + cmor.setup(inpath='./',netcdf_file_action=cmor.CMOR_REPLACE_4) #,logfile='cmorLog.txt') + cmor.dataset_json('tmp.json') + cmor.load_table(cmorTable) + #cmor.set_cur_dataset_attribute('history',f.history) ; # Force input file attribute as history + # Create axes based on variable + # Use height2m default value + heightAx = {'table_entry': 'height2m', + 'units': 'm', + 'coord_vals': cdm.createAxis([2.],id='height') + } + # Create time based on table + if key == 'A3hrPt': + timeAx = {'table_entry': 'time1', + 'units': time.units + } + else: + # Use default time entry + timeAx = {'table_entry': 'time', + 'units': time.units + } + # Now set axes list based on variable + if var in ['q_10','t_10','u_10','v_10']: + # 4D variable - 4 axes + axes = [ timeAx, + heightAx, + {'table_entry': 'latitude', + 'units': 'degrees_north', + 'coord_vals': lat[:], + 'cell_bounds': lat.getBounds() + }, + {'table_entry': 'longitude', + 'units': 'degrees_east', + 'coord_vals': lon[:], + 'cell_bounds': lon.getBounds() + }, + ] + else: + # 3D variable - 3 axes + axes = [ timeAx, + {'table_entry': 'latitude', + 'units': 'degrees_north', + 'coord_vals': lat[:], + 'cell_bounds': lat.getBounds() + }, + {'table_entry': 'longitude', + 'units': 'degrees_east', + 'coord_vals': lon[:], + 'cell_bounds': lon.getBounds() + }, + ] + axisIds = list() ; # Create list of axes and build these for each variable from axis components above + for axis in axes: + axisId = cmor.axis(**axis) + axisIds.append(axisId) + + # For use in debugging + #print 'axes:',axes + #print 'd.shape:',d.shape + #print 'outputVarName:',outputVarName + #print 'd.units:',d.units + #print 'd.missing:',d.missing + #pdb.set_trace() ; # Debug statement + + # Setup units and create variable to write using cmor - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute + d.units = outputUnits + if inputDict[key][var]['positive'] == 'down': + varid = cmor.variable(outputVarName,d.units,axisIds,missing_value=d.missing,positive='down') + else: + varid = cmor.variable(outputVarName,d.units,axisIds,missing_value=d.missing) ; # This is not trapping the correct missing_value + values = np.array(d[:],np.float32) + + # Append valid_min and valid_max to variable before writing - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute + #cmor.set_variable_attribute(varid,'valid_min','f',2.0) + #cmor.set_variable_attribute(varid,'valid_max','f',3.0) + + # Prepare variable for writing, then write and close file - see https://cmor.llnl.gov/mydoc_cmor3_api/#cmor_set_variable_attribute + cmor.set_deflate(varid,1,1,1) ; # shuffle=1,deflate=1,deflate_level=1 - Deflate options compress file data + print 'Start CMOR write..' + cmor.write(varid,values,time_vals=time,time_bnds=time.getBounds()) ; # Write variable with time axis + print 'End CMOR write..' + # Alteratively write in append mode + #for i in range(0,len(time),10): + # print i + # cmor.write(varid,values[i*10:(i+1)*10],time_vals=time[i*10:(i+1)*10],time_bnds=time.getBounds()[i*10:(i+1)*10]) ; # Write variable with time axis + f.close() + cmor.close() + # Cleanup + os.remove('tmp.json') \ No newline at end of file diff --git a/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Afx.json b/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Afx.json index 7b1bde0..85b1bd9 100644 --- a/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Afx.json +++ b/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Afx.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos land", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Afx" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Lday.json b/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Lday.json index 0524ab0..87464e6 100644 --- a/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Lday.json +++ b/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_Lday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"land", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Lday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_OyrC.json b/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_OyrC.json index 7e3e367..d5fa909 100644 --- a/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_OyrC.json +++ b/demo/MRI-JMA-JRA55-do-1-3-2/Tables/input4MIPs_OyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OyrC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hr.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hr.json index 3ceeab5..545f0e2 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hr.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hr.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_A3hr" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hrPt.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hrPt.json index c139e80..48d6b4f 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hrPt.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_A3hrPt.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_A3hrPt" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_CV.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_CV.json index 98b3e3b..6c58208 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_CV.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_CV.json @@ -87,6 +87,7 @@ "CNRM-Cerfacs":"CNRM (Centre National de Recherches Meteorologiques, Toulouse 31057, France), CERFACS (Centre Europeen de Recherche et de Formation Avancee en Calcul Scientifique, Toulouse 31100, France)", "IACETH":"Institute for Atmosphere and Climate, ETH Zurich, Zurich 8092, Switzerland", "IAMC":"Integrated Assessment Modeling Consortium (see www.globalchange.umd.edu/iamc/membership for complete membership). Mailing address: International Institute for Applied Systems Analysis (IIASA), Schlossplatz 1, A-2361 Laxenburg, Austria", + "IIS-UTokyo":"Institute of Industrial Science, The University of Tokyo, Hongo, Bunkyo-ku 113-8656, Japan", "ImperialCollege":"Imperial College London, South Kensington Campus, London SW7 2AZ, UK", "MOHC":"Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK", "MPI-B":"Max Planck Institute for Biogeochemistry, Jena 07745, Germany", @@ -240,6 +241,38 @@ "variable_id" ], "source_id":{ + "IIS-UTokyo-GSWP3-LS3MIP-1-1":{ + "comment":"Atmospheric boundary condition for retrospective off-line experiment of Global Soil Wetness Project Phase 3 (GSWP3, 1901-01 to 2014-12)", + "contact":"Hyungjun Kim (hjkim@iis.u-tokyo.ac.jp)", + "dataset_category":"atmosphericState", + "further_info_url":"https://hyungjun.github.io/GSWP3.DataDescription/; http://search.diasjp.net/en/dataset/GSWP3_EXP1_Forcing", + "grid":"0.5x0.5 degree latitude x longitude", + "grid_label":"gn", + "institution_id":"IIS-UTokyo", + "nominal_resolution":"50 km", + "product":"bias corrected & downscaled reanalysis", + "references":"Kim 2017: Global Soil Wetness Project Phase 3 Atmospheric Boundary Conditions (Experiment 1) [Data set]. Data Integration and Analysis System (DIAS). https://doi.org/10.20783/DIAS.501, ", + "region":[ + "global_land" + ], + "release_year":"2019", + "source":"GSWP3-EXP1 for LS3MIP-landhist (v1.1): Atmospheric boundary conditions for landhist experiment of LS3MIP", + "source_description":"Land state datasets produced by the University of Tokyo for the LS3MIP landhist experiment of CMIP6", + "source_id":"IIS-UTokyo-GSWP3-LS3MIP-1-1", + "source_variables":[ + "huss", + "prra", + "prsn", + "psl", + "rlds", + "rsds", + "sfcWind", + "tas" + ], + "source_version":"1.1", + "target_mip":"LS3MIP", + "title":"IIS-UTokyo GSWP3 LS3MIP 1.1 dataset prepared for input4MIPs" + }, "MRI-JRA55-do-1-3":{ "source":"MRI JRA55-do 1.3: Atmospheric state generated for OMIP based on the JRA-55 reanalysis" }, @@ -261,7 +294,6 @@ "source_id":"MRI-JRA55-do-1-3-2", "source_type":"satellite_blended", "source_variables":[ - "areacellg", "areacello", "friver", "huss", @@ -304,7 +336,6 @@ "source_id":"MRI-JRA55-do-1-4-0", "source_type":"satellite_blended", "source_variables":[ - "areacellg", "areacello", "friver", "huss", diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIday.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIday.json index 256b1bb..878128c 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIday.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIyrC.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIyrC.json index af0fc5c..a67c965 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIyrC.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_LIyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIyrC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_Oday.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_Oday.json index 6ef7425..da74a3a 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_Oday.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_Oday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Oday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OmonC.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OmonC.json index eb4ccb5..dc232e0 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OmonC.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OmonC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OmonC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OyrC.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OyrC.json index 7e3e367..d5fa909 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OyrC.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_OyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OyrC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SI3hrPt.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SI3hrPt.json index 114b394..64517f6 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SI3hrPt.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SI3hrPt.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SI3hrPt" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SIday.json b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SIday.json index e5effad..eb89404 100644 --- a/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SIday.json +++ b/demo/MRI-JMA-JRA55-do-1-3/Tables/input4MIPs_SIday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SIday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-3/demo.zip b/demo/MRI-JMA-JRA55-do-1-3/demo.zip index b04014a..e1003c3 100644 Binary files a/demo/MRI-JMA-JRA55-do-1-3/demo.zip and b/demo/MRI-JMA-JRA55-do-1-3/demo.zip differ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hr.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hr.json index 3ceeab5..545f0e2 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hr.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hr.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_A3hr" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hrPt.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hrPt.json index c139e80..48d6b4f 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hrPt.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_A3hrPt.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_A3hrPt" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Afx.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Afx.json index 7b1bde0..85b1bd9 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Afx.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Afx.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"atmos land", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Afx" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_CV.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_CV.json index badcd26..6c58208 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_CV.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_CV.json @@ -87,6 +87,7 @@ "CNRM-Cerfacs":"CNRM (Centre National de Recherches Meteorologiques, Toulouse 31057, France), CERFACS (Centre Europeen de Recherche et de Formation Avancee en Calcul Scientifique, Toulouse 31100, France)", "IACETH":"Institute for Atmosphere and Climate, ETH Zurich, Zurich 8092, Switzerland", "IAMC":"Integrated Assessment Modeling Consortium (see www.globalchange.umd.edu/iamc/membership for complete membership). Mailing address: International Institute for Applied Systems Analysis (IIASA), Schlossplatz 1, A-2361 Laxenburg, Austria", + "IIS-UTokyo":"Institute of Industrial Science, The University of Tokyo, Hongo, Bunkyo-ku 113-8656, Japan", "ImperialCollege":"Imperial College London, South Kensington Campus, London SW7 2AZ, UK", "MOHC":"Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK", "MPI-B":"Max Planck Institute for Biogeochemistry, Jena 07745, Germany", @@ -240,6 +241,38 @@ "variable_id" ], "source_id":{ + "IIS-UTokyo-GSWP3-LS3MIP-1-1":{ + "comment":"Atmospheric boundary condition for retrospective off-line experiment of Global Soil Wetness Project Phase 3 (GSWP3, 1901-01 to 2014-12)", + "contact":"Hyungjun Kim (hjkim@iis.u-tokyo.ac.jp)", + "dataset_category":"atmosphericState", + "further_info_url":"https://hyungjun.github.io/GSWP3.DataDescription/; http://search.diasjp.net/en/dataset/GSWP3_EXP1_Forcing", + "grid":"0.5x0.5 degree latitude x longitude", + "grid_label":"gn", + "institution_id":"IIS-UTokyo", + "nominal_resolution":"50 km", + "product":"bias corrected & downscaled reanalysis", + "references":"Kim 2017: Global Soil Wetness Project Phase 3 Atmospheric Boundary Conditions (Experiment 1) [Data set]. Data Integration and Analysis System (DIAS). https://doi.org/10.20783/DIAS.501, ", + "region":[ + "global_land" + ], + "release_year":"2019", + "source":"GSWP3-EXP1 for LS3MIP-landhist (v1.1): Atmospheric boundary conditions for landhist experiment of LS3MIP", + "source_description":"Land state datasets produced by the University of Tokyo for the LS3MIP landhist experiment of CMIP6", + "source_id":"IIS-UTokyo-GSWP3-LS3MIP-1-1", + "source_variables":[ + "huss", + "prra", + "prsn", + "psl", + "rlds", + "rsds", + "sfcWind", + "tas" + ], + "source_version":"1.1", + "target_mip":"LS3MIP", + "title":"IIS-UTokyo GSWP3 LS3MIP 1.1 dataset prepared for input4MIPs" + }, "MRI-JRA55-do-1-3":{ "source":"MRI JRA55-do 1.3: Atmospheric state generated for OMIP based on the JRA-55 reanalysis" }, diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIday.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIday.json index 256b1bb..878128c 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIday.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIyrC.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIyrC.json index af0fc5c..a67c965 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIyrC.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_LIyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"landIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_LIyrC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Lday.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Lday.json index 0524ab0..87464e6 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Lday.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Lday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"land", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Lday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Oday.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Oday.json index 6ef7425..da74a3a 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Oday.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_Oday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_Oday" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OmonC.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OmonC.json index eb4ccb5..dc232e0 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OmonC.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OmonC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OmonC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OyrC.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OyrC.json index 7e3e367..d5fa909 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OyrC.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_OyrC.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"ocean", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_OyrC" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SI3hrPt.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SI3hrPt.json index 114b394..64517f6 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SI3hrPt.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SI3hrPt.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SI3hrPt" }, "variable_entry":{ diff --git a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SIday.json b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SIday.json index e5effad..eb89404 100644 --- a/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SIday.json +++ b/demo/MRI-JMA-JRA55-do-1-4-0/Tables/input4MIPs_SIday.json @@ -10,7 +10,7 @@ "missing_value":"1e20", "product":"input4MIPs", "realm":"seaIce", - "table_date":"06 March 2019", + "table_date":"07 March 2019", "table_id":"Table input4MIPs_SIday" }, "variable_entry":{ diff --git a/input4MIPs_institution_id.json b/input4MIPs_institution_id.json index 25eb6cd..95b1d24 100644 --- a/input4MIPs_institution_id.json +++ b/input4MIPs_institution_id.json @@ -3,6 +3,7 @@ "CNRM-Cerfacs":"CNRM (Centre National de Recherches Meteorologiques, Toulouse 31057, France), CERFACS (Centre Europeen de Recherche et de Formation Avancee en Calcul Scientifique, Toulouse 31100, France)", "IACETH":"Institute for Atmosphere and Climate, ETH Zurich, Zurich 8092, Switzerland", "IAMC":"Integrated Assessment Modeling Consortium (see www.globalchange.umd.edu/iamc/membership for complete membership). Mailing address: International Institute for Applied Systems Analysis (IIASA), Schlossplatz 1, A-2361 Laxenburg, Austria", + "IIS-UTokyo":"Institute of Industrial Science, The University of Tokyo, Hongo, Bunkyo-ku 113-8656, Japan", "ImperialCollege":"Imperial College London, South Kensington Campus, London SW7 2AZ, UK", "MOHC":"Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK", "MPI-B":"Max Planck Institute for Biogeochemistry, Jena 07745, Germany", diff --git a/input4MIPs_source_id.json b/input4MIPs_source_id.json index 9b7d35c..02d3877 100644 --- a/input4MIPs_source_id.json +++ b/input4MIPs_source_id.json @@ -1,5 +1,37 @@ { "source_id":{ + "IIS-UTokyo-GSWP3-LS3MIP-1-1":{ + "comment":"Atmospheric boundary condition for retrospective off-line experiment of Global Soil Wetness Project Phase 3 (GSWP3, 1901-01 to 2014-12)", + "contact":"Hyungjun Kim (hjkim@iis.u-tokyo.ac.jp)", + "dataset_category":"atmosphericState", + "further_info_url":"https://hyungjun.github.io/GSWP3.DataDescription/; http://search.diasjp.net/en/dataset/GSWP3_EXP1_Forcing", + "grid":"0.5x0.5 degree latitude x longitude", + "grid_label":"gn", + "institution_id":"IIS-UTokyo", + "nominal_resolution":"50 km", + "product":"bias corrected & downscaled reanalysis", + "references":"Kim 2017: Global Soil Wetness Project Phase 3 Atmospheric Boundary Conditions (Experiment 1) [Data set]. Data Integration and Analysis System (DIAS). https://doi.org/10.20783/DIAS.501, ", + "region":[ + "global_land" + ], + "release_year":"2019", + "source":"GSWP3-EXP1 for LS3MIP-landhist (v1.1): Atmospheric boundary conditions for landhist experiment of LS3MIP", + "source_description":"Land state datasets produced by the University of Tokyo for the LS3MIP landhist experiment of CMIP6", + "source_id":"IIS-UTokyo-GSWP3-LS3MIP-1-1", + "source_variables":[ + "huss", + "prra", + "prsn", + "psl", + "rlds", + "rsds", + "sfcWind", + "tas" + ], + "source_version":"1.1", + "target_mip":"LS3MIP", + "title":"IIS-UTokyo GSWP3 LS3MIP 1.1 dataset prepared for input4MIPs" + }, "MRI-JRA55-do-1-3":{ "source":"MRI JRA55-do 1.3: Atmospheric state generated for OMIP based on the JRA-55 reanalysis" }, diff --git a/src/writeJson.py b/src/writeJson.py index 4c955a6..b31d189 100755 --- a/src/writeJson.py +++ b/src/writeJson.py @@ -65,12 +65,16 @@ PJD 23 Dec 2018 - Register source_id PCMDI-AMIP-1-1-5 https://github.com/PCMDI/input4MIPs-cmor-tables/issues/57 PJD 23 Dec 2018 - Revise source_id PCMDI-AMIP-1-1-4 PJD 23 Dec 2018 - Register institution_id MPI-B https://github.com/PCMDI/input4MIPs-cmor-tables/issues/59 +PJD 20 Jan 2019 - Register institution_id and source_id values for LS3MIP data and demo https://github.com/PCMDI/input4MIPs-cmor-tables/issues/62 PJD 30 Jan 2019 - Revise source_id MRI-JRA55-do-1-4, and generate LIday table https://github.com/PCMDI/input4MIPs-cmor-tables/issues/65 PJD 25 Feb 2019 - Register institution_id UCI https://github.com/PCMDI/input4MIPs-cmor-tables/issues/67 PJD 28 Feb 2019 - Amend MRI-JRA55-do source_id values; Update OyrC table entries https://github.com/PCMDI/input4MIPs-cmor-tables/issues/72 PJD 28 Feb 2019 - Update to include Afx table (areacella, sftlf) for MRI-JRA55-do https://github.com/PCMDI/input4MIPs-cmor-tables/issues/74 PJD 6 Mar 2019 - Tweaks required to correctly align variables with realms/Tables https://github.com/PCMDI/input4MIPs-cmor-tables/issues/81 PJD 6 Mar 2019 - Updated homePath +PJD 7 Mar 2019 - Merge branch with master - register institution_id and source_id values for LS3MIP data and demo https://github.com/PCMDI/input4MIPs-cmor-tables/issues/62 +PJD 7 Mar 2019 - Fighting with linux relative paths - will need to check MacOS + - TODO: Deal with lab cert issue https://raw.githubusercontent.com -> http://rawgit.com (see requests library) @author: durack1 @@ -501,6 +505,7 @@ institution_id['IAMC'] = ''.join(['Integrated Assessment Modeling Consortium (see www.globalchange.umd.edu/iamc/membership ', 'for complete membership). Mailing address: International Institute for Applied Systems Analysis ', '(IIASA), Schlossplatz 1, A-2361 Laxenburg, Austria']) +institution_id['IIS-UTokyo'] = 'Institute of Industrial Science, The University of Tokyo, Hongo, Bunkyo-ku 113-8656, Japan' institution_id['ImperialCollege'] = 'Imperial College London, South Kensington Campus, London SW7 2AZ, UK' institution_id['MOHC'] = 'Met Office Hadley Centre, Fitzroy Road, Exeter, Devon, EX1 3PB, UK' institution_id['MPI-B'] = 'Max Planck Institute for Biogeochemistry, Jena 07745, Germany' @@ -592,10 +597,32 @@ source_id = source_id.get('source_id') # Fix issues - -#key = 'MRI-JRA55-do-1-4' -#source_id.pop(key) - +key = 'IIS-UTokyo-GSWP3-LS3MIP-1-1' +source_id[key] = {} +source_id[key]['comment'] = 'Atmospheric boundary condition for retrospective off-line experiment of Global Soil Wetness Project Phase 3 (GSWP3, 1901-01 to 2014-12)' +source_id[key]['contact'] = 'Hyungjun Kim (hjkim@iis.u-tokyo.ac.jp)' +source_id[key]['dataset_category'] = 'atmosphericState' +source_id[key]['grid'] = '0.5x0.5 degree latitude x longitude' +source_id[key]['grid_label'] = 'gn' +source_id[key]['further_info_url'] = 'https://hyungjun.github.io/GSWP3.DataDescription/; http://search.diasjp.net/en/dataset/GSWP3_EXP1_Forcing' +source_id[key]['institution_id'] = 'IIS-UTokyo' +source_id[key]['nominal_resolution'] = '50 km' +source_id[key]['product'] = 'bias corrected & downscaled reanalysis' +source_id[key]['references'] = ''.join(['Kim 2017: Global Soil Wetness Project Phase 3 Atmospheric ', + 'Boundary Conditions (Experiment 1) [Data set]. Data ', + 'Integration and Analysis System (DIAS). ', + 'https://doi.org/10.20783/DIAS.501, ']) +source_id[key]['region'] = ['global_land'] +source_id[key]['release_year'] = '2019' +source_id[key]['source_description'] = 'Land state datasets produced by the University of Tokyo for the LS3MIP landhist experiment of CMIP6' +source_id[key]['source'] = 'GSWP3-EXP1 for LS3MIP-landhist (v1.1): Atmospheric boundary conditions for landhist experiment of LS3MIP' +source_id[key]['source_id'] = key +source_id[key]['source_variables'] = ['huss', 'prra', 'prsn', 'psl', 'rlds', 'rsds', 'sfcWind', 'tas'] +source_id[key]['source_version'] = '1.1' +source_id[key]['target_mip'] = 'LS3MIP' +source_id[key]['title'] = 'IIS-UTokyo GSWP3 LS3MIP 1.1 dataset prepared for input4MIPs' +#---- +''' key = 'MRI-JRA55-do-1-4-0' #source_id[key] = {} source_id[key]['comment'] = 'Based on JRA-55 reanalysis (1958-01 to 2019-01)' @@ -649,37 +676,37 @@ source_id[key]['source_version'] = '1.3.2' source_id[key]['target_mip'] = 'OMIP' source_id[key]['title'] = 'MRI JRA55-do 1.3.2 dataset prepared for input4MIPs' -#-- -#key = 'PCMDI-AMIP-1-1-5' -#source_id[key] = {} -#source_id[key]['comment'] = 'Based on Hurrell SST/sea ice consistency criteria applied to merged HadISST (1870-01 to 1981-10) & NCEP-0I2 (1981-11 to 2018-06)' -#source_id[key]['contact'] = 'PCMDI (pcmdi-cmip@llnl.gov)' -#source_id[key]['dataset_category'] = 'SSTsAndSeaIce' -#source_id[key]['grid'] = '1x1 degree longitude x latitude' -#source_id[key]['grid_label'] = 'gn' -#source_id[key]['further_info_url'] = 'https://pcmdi.llnl.gov/mips/amip' -#source_id[key]['institution_id'] = 'PCMDI' -#source_id[key]['institution'] = 'Program for Climate Model Diagnosis and Intercomparison, Lawrence Livermore National Laboratory, Livermore, CA 94550, USA' -#source_id[key]['nominal_resolution'] = '1x1 degree' -#source_id[key]['product'] = 'observations' -#source_id[key]['references'] = ''.join(['Taylor, K.E., D. Williamson and F. Zwiers, ', -# '2000: The sea surface temperature and sea ice ', -# 'concentration boundary conditions for AMIP II ', -# 'simulations. PCMDI Report 60, Program for ', -# 'Climate Model Diagnosis and Intercomparison, ', -# 'Lawrence Livermore National Laboratory, 25 pp. ', -# 'Available online: https://pcmdi.llnl.gov/report/pdf/60.pdf']) -#source_id[key]['region'] = ['global_ocean'] -#source_id[key]['release_year'] = '2018' -#source_id[key]['source_description'] = 'Sea surface temperature and sea-ice datasets produced by PCMDI (LLNL) for the AMIP (DECK) experiment of CMIP6' -#source_id[key]['source'] = 'PCMDI-AMIP 1.1.5: Merged SST based on UK MetOffice HadISST and NCEP OI2' -#source_id[key]['source_id'] = key -#source_id[key]['source_type'] = 'satellite_blended' -#source_id[key]['source_variables'] = ['areacello','sftof','siconc','siconcbcs', -# 'tos','tosbcs'] -#source_id[key]['source_version'] = '1.1.5' -#source_id[key]['target_mip'] = 'CMIP' -#source_id[key]['title'] = 'PCMDI-AMIP 1.1.5 dataset prepared for input4MIPs' + +key = 'PCMDI-AMIP-1-1-5' +source_id[key] = {} +source_id[key]['comment'] = 'Based on Hurrell SST/sea ice consistency criteria applied to merged HadISST (1870-01 to 1981-10) & NCEP-0I2 (1981-11 to 2018-06)' +source_id[key]['contact'] = 'PCMDI (pcmdi-cmip@llnl.gov)' +source_id[key]['dataset_category'] = 'SSTsAndSeaIce' +source_id[key]['grid'] = '1x1 degree longitude x latitude' +source_id[key]['grid_label'] = 'gn' +source_id[key]['further_info_url'] = 'https://pcmdi.llnl.gov/mips/amip' +source_id[key]['institution_id'] = 'PCMDI' +source_id[key]['institution'] = 'Program for Climate Model Diagnosis and Intercomparison, Lawrence Livermore National Laboratory, Livermore, CA 94550, USA' +source_id[key]['nominal_resolution'] = '1x1 degree' +source_id[key]['product'] = 'observations' +source_id[key]['references'] = ''.join(['Taylor, K.E., D. Williamson and F. Zwiers, ', + '2000: The sea surface temperature and sea ice ', + 'concentration boundary conditions for AMIP II ', + 'simulations. PCMDI Report 60, Program for ', + 'Climate Model Diagnosis and Intercomparison, ', + 'Lawrence Livermore National Laboratory, 25 pp. ', + 'Available online: https://pcmdi.llnl.gov/report/pdf/60.pdf']) +source_id[key]['region'] = ['global_ocean'] +source_id[key]['release_year'] = '2018' +source_id[key]['source_description'] = 'Sea surface temperature and sea-ice datasets produced by PCMDI (LLNL) for the AMIP (DECK) experiment of CMIP6' +source_id[key]['source'] = 'PCMDI-AMIP 1.1.5: Merged SST based on UK MetOffice HadISST and NCEP OI2' +source_id[key]['source_id'] = key +source_id[key]['source_type'] = 'satellite_blended' +source_id[key]['source_variables'] = ['areacello','sftof','siconc','siconcbcs', + 'tos','tosbcs'] +source_id[key]['source_version'] = '1.1.5' +source_id[key]['target_mip'] = 'CMIP' +''' #%% Create CV master CV = {} @@ -752,6 +779,7 @@ # Validate - only necessary if files are not written by json module #%% Incorporate JSON versioning info - see https://docs.google.com/document/d/1pU9IiJvPJwRvIgVaSDdJ4O0Jeorv_2ekEtted34K9cA/edit#heading=h.w4kchhc266o3 +print 'Start Versions write:',os.getcwd() versionId = '6.2.3' input4MIPs = {} input4MIPs['data'] = {} @@ -841,13 +869,14 @@ input4MIPs['data']['CMIP']['VUA']['emissions']['currentVersion'] = '1.2' input4MIPs['data']['CMIP']['VUA']['emissions']['deprecatedVersion'] = '1.0' # Write version file -outFile = ''.join(['../Versions/',versionId,'.json']) +print 'Start Versions write:',os.getcwd() +outFile = ''.join(['Versions/',versionId,'.json']) # Check file exists if os.path.exists(outFile): print('File existing, purging:',outFile) os.remove(outFile) -if not os.path.exists('../Versions'): - os.mkdir('../Versions') +if not os.path.exists('Versions'): + os.mkdir('Versions') # Create host dictionary jsonDict = {} jsonDict['input4MIPs_version'] = {} @@ -858,24 +887,30 @@ fH.close() #%% Generate MRI-JMA-JRA55-do-1-3 demo directory -demoPath = os.path.join('/','/'.join(os.path.realpath(__file__).split('/')[0:-2]),'demo') +print 'Start MRI-JMA-JRA55-do-1-3 write:',os.getcwd() +demoPath = os.path.join('/','/'.join(os.path.realpath(__file__).split('/')[0:-1]),'demo') +print 'demoPath:',demoPath demoPath = os.path.join(demoPath,'MRI-JMA-JRA55-do-1-3') +print 'demoPath:',demoPath outPath = os.path.join(demoPath,'Tables') +print 'outPath:',outPath # First purge existing if os.path.exists(outPath): shutil.rmtree(outPath) ; # Purge all existing os.makedirs(outPath) else: os.makedirs(outPath) -os.chdir(demoPath) +os.chdir(outPath) +print 'Write MRI-JMA-JRA55-do-1-3 write:',os.getcwd() # Now fill Tables subdir with required files cvTables = ['A3hr','A3hrPt','CV','Oday','OmonC','OyrC','SI3hrPt','LIday', 'LIyrC','SIday','coordinate','formula_terms'] for count,tableId in enumerate(cvTables): fileName = ''.join(['input4MIPs_',tableId,'.json']) - sourcePath = os.path.join('..','..','Tables',fileName) - shutil.copy(sourcePath,'Tables') + print 'Write loop MRI-JMA-JRA55-do-1-3:',os.getcwd() + sourcePath = os.path.join('..','..','..','Tables',fileName) + shutil.copy(sourcePath,'.') #%% Generate zip archive env7za = os.environ.copy() @@ -903,6 +938,7 @@ shutil.move('demo.zip', 'MRI-JMA-JRA55-do-1-3/demo.zip') #%% Generate MRI-JMA-JRA55-do-1-3-2 demo directory +print 'Start MRI-JMA-JRA55-do-1-3-2 write:',os.getcwd() #demoPath = os.path.join('/','/'.join(os.path.realpath(__file__).split('/')[0:-2]),'demo') demoPath = os.path.join(homePath,'demo') os.chdir(demoPath) @@ -922,10 +958,12 @@ #print 'MRI-JMA-JRA55-do-1-3-2 demo:',os.getcwd() for count,tableId in enumerate(cvTables): fileName = ''.join(['input4MIPs_',tableId,'.json']) - sourcePath = os.path.join('..','..','..','..','Tables',fileName) + print 'Write loop MRI-JMA-JRA55-do-1-3-2:',os.getcwd() + sourcePath = os.path.join('..','..','..','Tables',fileName) shutil.copy(sourcePath,'.') #%% Generate MRI-JMA-JRA55-do-1-4-0 demo directory +print 'Start MRI-JMA-JRA55-do-1-4-0 write:',os.getcwd() demoPath = os.path.join(homePath,'demo') os.chdir(demoPath) #print 'MRI-JMA-JRA55-do-1-4-0 demo:',os.getcwd() @@ -945,5 +983,6 @@ #print 'MRI-JMA-JRA55-do-1-4-0 demo:',os.getcwd() for count,tableId in enumerate(cvTables): fileName = ''.join(['input4MIPs_',tableId,'.json']) + print 'Write loop MRI-JMA-JRA55-do-1-4-0:',os.getcwd() sourcePath = os.path.join('..','..','..','Tables',fileName) shutil.copy(sourcePath,'.') \ No newline at end of file